{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Rate Setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torchvision.models as models\n",
    "\n",
    "from torch import optim\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.resnet18()\n",
    "optimizer = optim.SGD(params = model.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.不设置, 保持学习率不变"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x118b568d0>]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADo9JREFUeJzt23+s3XV9x/HnS0p1/lqBNgTbjmLsNqtxwq4Icwphi2vZRidZNogJP2LSP4TMLSMLhD+IGGM23ebIDKTTDqsLzDHnqmNDxo/wjzBuB1SgAhc27S3VXoPgGH8g+t4f51tyvOvtub09hwPn83wkJz3fH/eczyef5nnP+Z5zU1VIktrwqnEPQJL00jH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDVk27gHMt3Llylq3bt24hyFJryg7d+78flWtGnTeyy7669atY3p6etzDkKRXlCTfXsx5Xt6RpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqyMDoJ9mWZH+SBxc4niTXJJlJsivJKfOOvzHJbJK/HtagJUlLs5hX+tcDGw9xfBOwvrttAa6dd/xjwF1LGZwkabgGRr+q7gKeOsQpm4Ht1XM3sCLJCQBJfhk4Hvj6MAYrSToyw7imvxrY07c9C6xO8irgz4HLhvAckqQhGOUHuR8Gbq6q2UEnJtmSZDrJ9Nzc3AiHJEltWzaEx9gLrO3bXtPtOx14b5IPA68Hlid5tqoun/8AVbUV2AowNTVVQxiTJOkghhH9HcClSW4E3g08U1X7gA8eOCHJRcDUwYIvSXrpDIx+khuAM4GVSWaBq4CjAarqOuBm4GxgBngOuHhUg5UkHZmB0a+q8wccL+CSAedcT++rn5KkMfIvciWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIUZfkhpi9CWpIQOjn2Rbkv1JHlzgeJJck2Qmya4kp3T735nkG0ke6vb//rAHL0k6PIt5pX89sPEQxzcB67vbFuDabv9zwAVV9bbu5z+dZMXShypJOlLLBp1QVXclWXeIUzYD26uqgLuTrEhyQlU92vcYTybZD6wCnj7CMUuSlmgY1/RXA3v6tme7fS9KciqwHHh8CM8nSVqikX+Qm+QE4AvAxVX1kwXO2ZJkOsn03NzcqIckSc0aRvT3Amv7ttd0+0jyRuBfgCur6u6FHqCqtlbVVFVNrVq1aghDkiQdzDCivwO4oPsWz2nAM1W1L8ly4J/oXe+/aQjPI0k6QgM/yE1yA3AmsDLJLHAVcDRAVV0H3AycDczQ+8bOxd2P/h7wPuC4JBd1+y6qqvuHOH5J0mFYzLd3zh9wvIBLDrL/i8AXlz40SdKw+Re5ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDTH6ktQQoy9JDRkY/STbkuxP8uACx5PkmiQzSXYlOaXv2IVJHutuFw5z4JKkw7eYV/rXAxsPcXwTsL67bQGuBUhyLHAV8G7gVOCqJMccyWAlSUdmYPSr6i7gqUOcshnYXj13AyuSnAD8BnBrVT1VVT8AbuXQvzwkSSO2bAiPsRrY07c92+1baP/IfPSrD/Hwkz8c5VNI0shseNMbueq33zbS53hZfJCbZEuS6STTc3Nz4x6OJE2sYbzS3wus7dte0+3bC5w5b/+dB3uAqtoKbAWYmpqqpQ5k1L8hJemVbhiv9HcAF3Tf4jkNeKaq9gG3AO9Pckz3Ae77u32SpDEZ+Eo/yQ30XrGvTDJL7xs5RwNU1XXAzcDZwAzwHHBxd+ypJB8D7u0e6uqqOtQHwpKkERsY/ao6f8DxAi5Z4Ng2YNvShiZJGraXxQe5kqSXhtGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqiNGXpIYYfUlqyKKin2RjkkeSzCS5/CDHT0xyW5JdSe5Msqbv2J8leSjJ7iTXJMkwJyBJWryB0U9yFPAZYBOwATg/yYZ5p30K2F5V7wCuBj7R/eyvAO8B3gG8HXgXcMbQRi9JOiyLeaV/KjBTVU9U1fPAjcDmeedsAG7v7t/Rd7yA1wDLgVcDRwPfO9JBS5KWZjHRXw3s6due7fb1ewA4t7v/AeANSY6rqm/Q+yWwr7vdUlW7j2zIkqSlGtYHuZcBZyS5j97lm73Aj5O8BXgrsIbeL4qzkrx3/g8n2ZJkOsn03NzckIYkSZpvMdHfC6zt217T7XtRVT1ZVedW1cnAld2+p+m96r+7qp6tqmeBfwVOn/8EVbW1qqaqamrVqlVLnIokaZDFRP9eYH2Sk5IsB84DdvSfkGRlkgOPdQWwrbv/HXrvAJYlOZreuwAv70jSmAyMflW9AFwK3EIv2F+qqoeSXJ3knO60M4FHkjwKHA98vNt/E/A48E161/0fqKqvDncKkqTFSlWNeww/ZWpqqqanp8c9DEl6RUmys6qmBp3nX+RKUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1xOhLUkOMviQ1ZFHRT7IxySNJZpJcfpDjJya5LcmuJHcmWdN37OeSfD3J7iQPJ1k3vOFLkg7HwOgnOQr4DLAJ2ACcn2TDvNM+BWyvqncAVwOf6Du2HfhkVb0VOBXYP4yBS5IO32Je6Z8KzFTVE1X1PHAjsHneORuA27v7dxw43v1yWFZVtwJU1bNV9dxQRi5JOmyLif5qYE/f9my3r98DwLnd/Q8Ab0hyHPDzwNNJvpzkviSf7N45SJLGYFgf5F4GnJHkPuAMYC/wY2AZ8N7u+LuANwMXzf/hJFuSTCeZnpubG9KQJEnzLSb6e4G1fdtrun0vqqonq+rcqjoZuLLb9zS9dwX3d5eGXgC+Apwy/wmqamtVTVXV1KpVq5Y4FUnSIIuJ/r3A+iQnJVkOnAfs6D8hycokBx7rCmBb38+uSHKg5GcBDx/5sCVJSzEw+t0r9EuBW4DdwJeq6qEkVyc5pzvtTOCRJI8CxwMf7372x/Qu7dyW5JtAgL8Z+iwkSYuSqhr3GH7K1NRUTU9Pj3sYkvSKkmRnVU0NOs+/yJWkhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0Zekhhh9SWqI0ZekhqSqxj2Gn5JkDvj2ETzESuD7QxrOK0WLc4Y2593inKHNeR/unE+sqlWDTnrZRf9IJZmuqqlxj+Ol1OKcoc15tzhnaHPeo5qzl3ckqSFGX5IaMonR3zruAYxBi3OGNufd4pyhzXmPZM4Td01fkrSwSXylL0lawMREP8nGJI8kmUly+bjHMypJ1ia5I8nDSR5K8pFu/7FJbk3yWPfvMeMe67AlOSrJfUm+1m2flOSebs3/PsnycY9x2JKsSHJTkm8l2Z3k9Elf6yR/1P3ffjDJDUleM4lrnWRbkv1JHuzbd9C1Tc813fx3JTllqc87EdFPchTwGWATsAE4P8mG8Y5qZF4A/riqNgCnAZd0c70cuK2q1gO3dduT5iPA7r7tPwX+sqreAvwA+NBYRjVafwX8W1X9IvBL9OY/sWudZDXwB8BUVb0dOAo4j8lc6+uBjfP2LbS2m4D13W0LcO1Sn3Qiog+cCsxU1RNV9TxwI7B5zGMaiaraV1X/2d3/H3oRWE1vvp/vTvs88DvjGeFoJFkD/Cbw2W47wFnATd0pkzjnnwXeB3wOoKqer6qnmfC1BpYBP5NkGfBaYB8TuNZVdRfw1LzdC63tZmB79dwNrEhywlKed1KivxrY07c92+2baEnWAScD9wDHV9W+7tB3gePHNKxR+TTwJ8BPuu3jgKer6oVuexLX/CRgDvjb7rLWZ5O8jgle66raC3wK+A692D8D7GTy1/qAhdZ2aI2blOg3J8nrgX8E/rCqfth/rHpfyZqYr2Ul+S1gf1XtHPdYXmLLgFOAa6vqZOB/mXcpZwLX+hh6r2pPAt4EvI7/fwmkCaNa20mJ/l5gbd/2mm7fREpyNL3g/11Vfbnb/b0Db/e6f/ePa3wj8B7gnCT/Te/S3Vn0rnWv6C4BwGSu+SwwW1X3dNs30fslMMlr/evAf1XVXFX9CPgyvfWf9LU+YKG1HVrjJiX69wLru0/4l9P74GfHmMc0Et217M8Bu6vqL/oO7QAu7O5fCPzzSz22UamqK6pqTVWto7e2t1fVB4E7gN/tTpuoOQNU1XeBPUl+odv1a8DDTPBa07usc1qS13b/1w/MeaLXus9Ca7sDuKD7Fs9pwDN9l4EOT1VNxA04G3gUeBy4ctzjGeE8f5XeW75dwP3d7Wx617hvAx4D/h04dtxjHdH8zwS+1t1/M/AfwAzwD8Crxz2+Ecz3ncB0t95fAY6Z9LUGPgp8C3gQ+ALw6klca+AGep9b/Ijeu7oPLbS2QOh9Q/Fx4Jv0vt20pOf1L3IlqSGTcnlHkrQIRl+SGmL0JakhRl+SGmL0JakhRl+SGmL0JakhRl+SGvJ/nBh2mrGxcxkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ２.阶梯下降方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### (1)　公式 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming optimizer uses lr = 1.0 for all groups\n",
    "# lr = 1.0     if epoch < 30\n",
    "# lr = 0.1    if 30 <= epoch < 60\n",
    "# lr = 0.01   if 60 <= epoch < 90\n",
    "# ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x118bfbfd0>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAE91JREFUeJzt3X+QXWV9x/H3956zgGAEMVuQhBCEoKaOVmaLODqVUewA1aQz1hamHW1LjTOVaqvTDo4dtPSPjrVjW1uqTZWitoWidWxqY2mLtNoqSPAHhSC4IkooSETEX0WS8O0f92y4bO69e03u7t3n7Ps1s5N7zj3Z+5w5m0+e/Z7nPE9kJpKkdulMugGSpPEz3CWphQx3SWohw12SWshwl6QWMtwlqYUMd0lqIcNdklrIcJekFqon9cGrV6/O9evXT+rjJalIN9100zczc3qh4yYW7uvXr2fHjh2T+nhJKlJEfG2U4yzLSFILGe6S1EKGuyS1kOEuSS1kuEtSCy0Y7hFxeUTcHxG3DHg/IuJdETEbETdHxOnjb6Yk6UcxSs/9CuCcIe+fC2xovrYA7z70ZkmSDsWC49wz85MRsX7IIZuBD2R3vb7rI+KYiHhqZt47pjY+zo13fYtP3bF7Mb61eqw6YopffeHJVJ2YdFMkHYRxPMS0Bri7Z3tXs++AcI+ILXR796xbt+6gPuxzX3uQP7tu9qD+rkYzt6zu8095Cs9ac/RkGyPpoCzpE6qZuRXYCjAzM3NQK3O/9kWn8NoXnTLWdunx/vOO3bz68s/yyL5HJ90USQdpHKNl7gFO7Nle2+xToeqmFLN330H9/ytpGRhHuG8DXtWMmjkTeGix6u1aGnN19r2P2nOXSrVgWSYirgTOAlZHxC7grcAUQGa+B9gOnAfMAj8AfmWxGqulMVXZc5dKN8pomQsWeD+B142tRZq4qtP9hW7fo4a7VCqfUNUB9tfcDXepWIa7DlDvL8tYc5dKZbjrAPbcpfIZ7jpA3dTcHS0jlctw1wEqx7lLxTPcdYCpaq7nbrhLpTLcdYDKmrtUPMNdB5h7iGmfo2WkYhnuOoA9d6l8hrsO8NhoGcNdKpXhrgP4EJNUPsNdB/AhJql8hrsOEBFUnXDiMKlghrv6qjrBHh9ikopluKuvuhPsc/oBqViGu/qq7blLRTPc1Vddday5SwUz3NVX3QlnhZQKZrirr7oTzgopFcxwV19V5VBIqWSGu/qa6nTYY7hLxTLc1VflUEipaIa7+qqrjkMhpYIZ7uqrdvoBqWiGu/qqq2CPs0JKxTLc1Zc9d6lshrv6qjrhlL9SwQx39TVVdVysQyqY4a6+nM9dKpvhrr7qjkMhpZKNFO4RcU5E3B4RsxFxcZ/310XEdRHx+Yi4OSLOG39TtZS8oSqVbcFwj4gKuAw4F9gIXBARG+cd9rvA1Zn5XOB84C/G3VAtrapyVkipZKP03M8AZjPzzsx8BLgK2DzvmASe1Lw+Gvjf8TVRkzDlaBmpaPUIx6wB7u7Z3gU8b94xbwP+NSJ+AzgKOHssrdPEVJ2OU/5KBRvXDdULgCsycy1wHvDBiDjge0fElojYERE7du/ePaaP1mKYsiwjFW2UcL8HOLFne22zr9eFwNUAmfkZ4Ahg9fxvlJlbM3MmM2emp6cPrsVaEg6FlMo2SrjfCGyIiJMj4jC6N0y3zTvm68BLACLimXTD3a55waacFVIq2oLhnpl7gYuAa4Db6I6KuTUiLo2ITc1hbwJeExFfBK4EfjkzTYaC2XOXyjbKDVUyczuwfd6+S3pe7wReMN6maZJcIFsqm0+oqq+6coFsqWSGu/qqOh32PppYXZPKZLirr6lOAFh3lwpluKuvquqGu0+pSmUy3NVX3THcpZIZ7uqr7nR/NPZ5U1UqkuGuvur9ZRmHQ0olMtzV11zP3bKMVCbDXX1Zc5fKZrirr/1lGRfJlopkuKuvyp67VDTDXX3tHy1juEtFMtzV11xZZo9lGalIhrv6qp1+QCqa4a6+6qr7o+GCHVKZDHf1Zc9dKpvhrr72j5ax5i4VyXBXX1POCikVzXBXX5VDIaWiGe7qa67m7lBIqUyGu/qaG+duz10qk+GuvuaeUN1juEtFMtzV12NDIS3LSCUy3NVXtb/mbs9dKpHhrr6mKkfLSCUz3NWXU/5KZTPc1deUi3VIRTPc1Vfl3DJS0Qx39bV/KKQ3VKUiGe7q67GHmCzLSCUy3NVX7Q1VqWgjhXtEnBMRt0fEbERcPOCYn4+InRFxa0T83XibqaUWEVSdYK9lGalI9UIHREQFXAa8FNgF3BgR2zJzZ88xG4A3Ay/IzAcj4scWq8FaOlUn7LlLhRql534GMJuZd2bmI8BVwOZ5x7wGuCwzHwTIzPvH20xNQt0Jh0JKhRol3NcAd/ds72r29ToNOC0i/jsiro+Ic/p9o4jYEhE7ImLH7t27D67FWjK1PXepWOO6oVoDG4CzgAuAv4qIY+YflJlbM3MmM2emp6fH9NFaLHXVYa+jZaQijRLu9wAn9myvbfb12gVsy8w9mflV4A66Ya+C1Z3wISapUKOE+43Ahog4OSIOA84Hts075qN0e+1ExGq6ZZo7x9hOTUDtaBmpWAuGe2buBS4CrgFuA67OzFsj4tKI2NQcdg3wQETsBK4DfjszH1isRmtpdMsyhrtUogWHQgJk5nZg+7x9l/S8TuCNzZdawhuqUrl8QlUDVQ6FlIpluGsgyzJSuQx3DeRDTFK5DHcNVFfW3KVSGe4ayHHuUrkMdw3krJBSuQx3DTTl9ANSsQx3DeSUv1K5DHcNVHc6lmWkQhnuGsgbqlK5DHcNVFfBHmvuUpEMdw1kz10ql+GugSpr7lKxDHcNNFWFQyGlQhnuGsiHmKRyGe4aaMpZIaViGe4aqPKGqlQsw10D1Z1gj1P+SkUy3DVQXdlzl0pluGugqtOtuXeXyJVUEsNdA011AsDeu1Qgw10DVVU33B0xI5XHcNdAU53uj4fhLpXHcNdA1VxZxgeZpOIY7hqobsoyzgwplcdw10B1U5bxhqpUHsNdA9VNWcYHmaTyGO4aaK4sY89dKo/hroHmbqg6WkYqj+GugeZq7k77K5VnpHCPiHMi4vaImI2Ii4cc94qIyIiYGV8TNSn1/oeYrLlLpVkw3COiAi4DzgU2AhdExMY+x60C3gDcMO5GajLmbqjac5fKM0rP/QxgNjPvzMxHgKuAzX2O+33g7cDDY2yfJqiufEJVKtUo4b4GuLtne1ezb7+IOB04MTP/eYxt04Q91nO3LCOV5pBvqEZEB3gn8KYRjt0SETsiYsfu3bsP9aO1yGpnhZSKNUq43wOc2LO9ttk3ZxXwLOA/IuIu4ExgW7+bqpm5NTNnMnNmenr64FutJVE7K6RUrFHC/UZgQ0ScHBGHAecD2+bezMyHMnN1Zq7PzPXA9cCmzNyxKC3Wkqn2zwppWUYqzYLhnpl7gYuAa4DbgKsz89aIuDQiNi12AzU5jpaRylWPclBmbge2z9t3yYBjzzr0Zmk5sCwjlcsnVDVQ7WIdUrEMdw302GgZa+5SaQx3DVTtn/LXnrtUGsNdA01VLtYhlcpw10CVT6hKxTLcNdCUo2WkYhnuGqhynLtULMNdAzkUUiqX4a6BHltD1Zq7VBrDXQPVDoWUimW4a6CIoOqEQyGlAhnuGqrqBHssy0jFMdw11FQn2GdZRiqO4a6hqk44WkYqkOGuoeqq42IdUoEMdw1Ve0NVKpLhrqHqTjgUUiqQ4a6h6qpjz10qkOGuobo9d2vuUmkMdw3lQ0xSmQx3DdUdLWO4S6Ux3DVU3QkX65AKZLhrqLryISapRIa7hur23A13qTSGu4aqOw6FlEpkuGuounJWSKlEhruGciikVCbDXUPVnY41d6lAhruGqjvhrJBSgQx3DeVQSKlMhruGciikVKaRwj0izomI2yNiNiIu7vP+GyNiZ0TcHBHXRsRJ42+qJqFyKKRUpAXDPSIq4DLgXGAjcEFEbJx32OeBmcx8NvBh4A/H3VBNxlTlrJBSiUbpuZ8BzGbmnZn5CHAVsLn3gMy8LjN/0GxeD6wdbzM1KQ6FlMo0SrivAe7u2d7V7BvkQuDjh9IoLR9TzgopFake5zeLiF8CZoAXDXh/C7AFYN26deP8aC2SylkhpSKN0nO/BzixZ3tts+9xIuJs4C3Apsz8Yb9vlJlbM3MmM2emp6cPpr1aYg6FlMo0SrjfCGyIiJMj4jDgfGBb7wER8VzgL+kG+/3jb6YmpfsQk+EulWbBcM/MvcBFwDXAbcDVmXlrRFwaEZuaw94BPBH4UER8ISK2Dfh2KszcUMhMA14qyUg198zcDmyft++Sntdnj7ldWiamOgHAvkeTuooJt0bSqHxCVUNVTaBbmpHKYrhrqKlO90fEcJfKYrhrqKopyzgcUiqL4a6hassyUpEMdw1Vz5VlnBlSKorhrqHqubKMC3ZIRTHcNdRcWcbJw6SyGO4aau6G6h7LMlJRDHcNNVV1f0TsuUtlMdw11GM9d2vuUkkMdw1Vd6y5SyUy3DVUXc09oWrPXSqJ4a6h9g+F9IaqVJSxrsSk9pkL94/fch9fuu+7E25Ne1Wd4OXPPoGjj5yadFPUEoa7hjruSUdQd4IrPn3XpJvSet95eA+/ftapk26GWsJw11DrVx/FF9/60/xwrzX3xfSyd32K2/3NSGNkuGtBRx1ec9Thk25Fuz39+FWGu8bKG6rSMnDa8au4c/f3fZ5AY2O4S8vAM45fxSP7HuWub35/0k1RSxju0jJw2nGrALj9G5ZmNB6Gu7QMnDL9RDoBd1h315gY7tIycMRUxfrVR9lz19gY7tIy8fTjVnHHN7436WaoJQx3aZk47bhV3PXA93l4z75JN0UtYLhLy8Qzjl9FJszeb+9dh85wl5aJ047vjphxDh+Ng+EuLRMnHXskh9Ud7vCmqsbAcJeWibrqcOr0E52GQGNhuEvLyNOPX2XPXWNhuEvLyGnHreLehx7mof/bM+mmqHAjhXtEnBMRt0fEbERc3Of9wyPi75v3b4iI9eNuqLQSPKO5qfple+86RAtO+RsRFXAZ8FJgF3BjRGzLzJ09h10IPJiZp0bE+cDbgV9YjAZLbTY3YuYd19zOSU85csKtabeZ9cfyitPXUjWrjbXNKPO5nwHMZuadABFxFbAZ6A33zcDbmtcfBv48IiIzXXhT+hGccPQRvPDU1Xxl9/f4+rd+MOnmtNaefY9y9Y5dvP/Td/G2TT/OT64/dtJNGrtRwn0NcHfP9i7geYOOycy9EfEQ8BTgm+NopLRSRAR/82vz/3lp3DKTf7r5Xv5g+2288j2f4WnTR1HF0vXgX/+SDbz8OScs6mcs6UpMEbEF2AKwbt26pfxoSdovItj0nBN46TOP433/dSc77/3Okn7+0U9Y/IXQRwn3e4ATe7bXNvv6HbMrImrgaOCB+d8oM7cCWwFmZmYs2UiaqCccVnHRizdMuhmLYpTRMjcCGyLi5Ig4DDgf2DbvmG3Aq5vXPwd8wnq7JE3Ogj33poZ+EXANUAGXZ+atEXEpsCMztwHvAz4YEbPAt+j+ByBJmpCRau6ZuR3YPm/fJT2vHwZeOd6mSZIOlk+oSlILGe6S1EKGuyS1kOEuSS1kuEtSC8WkhqNHxG7gawf511ezMqc2WInnvRLPGVbmea/Ec4Yf/bxPyszphQ6aWLgfiojYkZkzk27HUluJ570SzxlW5nmvxHOGxTtvyzKS1EKGuyS1UKnhvnXSDZiQlXjeK/GcYWWe90o8Z1ik8y6y5i5JGq7UnrskaYjiwn2hxbrbICJOjIjrImJnRNwaEW9o9h8bEf8WEV9u/nzypNs6bhFRRcTnI+JjzfbJzaLrs80i7IdNuo3jFhHHRMSHI+JLEXFbRDx/hVzr32p+vm+JiCsj4oi2Xe+IuDwi7o+IW3r29b220fWu5txvjojTD+Wziwr3nsW6zwU2AhdExMbJtmpR7AXelJkbgTOB1zXneTFwbWZuAK5tttvmDcBtPdtvB/44M08FHqS7GHvb/CnwL5n5DOA5dM+/1dc6ItYArwdmMvNZdKcTP5/2Xe8rgHPm7Rt0bc8FNjRfW4B3H8oHFxXu9CzWnZmPAHOLdbdKZt6bmZ9rXn+X7j/2NXTP9f3NYe8HfnYyLVwcEbEW+Bngvc12AC+mu+g6tPOcjwZ+iu6aCGTmI5n5bVp+rRs18IRm9bYjgXtp2fXOzE/SXeOi16Bruxn4QHZdDxwTEU892M8uLdz7Lda9ZkJtWRIRsR54LnADcFxm3tu8dR9w3ISatVj+BPgd4NFm+ynAtzNzb7Pdxut9MrAb+OumHPXeiDiKll/rzLwH+CPg63RD/SHgJtp/vWHwtR1rvpUW7itKRDwR+AfgNzPzcSv4NssYtmaoU0S8DLg/M2+adFuWWA2cDrw7M58LfJ95JZi2XWuAps68me5/bicAR3Fg+aL1FvPalhbuoyzW3QoRMUU32P82Mz/S7P7G3K9pzZ/3T6p9i+AFwKaIuItuue3FdGvRxzS/tkM7r/cuYFdm3tBsf5hu2Lf5WgOcDXw1M3dn5h7gI3R/Btp+vWHwtR1rvpUW7qMs1l28ptb8PuC2zHxnz1u9C5G/GvjHpW7bYsnMN2fm2sxcT/e6fiIzfxG4ju6i69CycwbIzPuAuyPi6c2ulwA7afG1bnwdODMijmx+3ufOu9XXuzHo2m4DXtWMmjkTeKinfPOjy8yivoDzgDuArwBvmXR7FukcX0j3V7WbgS80X+fRrUFfC3wZ+Hfg2Em3dZHO/yzgY83rpwGfBWaBDwGHT7p9i3C+PwHsaK73R4Enr4RrDfwe8CXgFuCDwOFtu97AlXTvKeyh+1vahYOuLRB0RwN+BfgfuiOJDvqzfUJVklqotLKMJGkEhrsktZDhLkktZLhLUgsZ7pLUQoa7JLWQ4S5JLWS4S1IL/T9zcuRV5ENJmgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)\n",
    "\n",
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "    scheduler.step()\n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### (2)　公式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x118ccf510>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD8CAYAAABpcuN4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF71JREFUeJzt3X+QXeV93/H35567UoyhwpbXsq0fkRoteJY4iTNr7LaZloYwiMQTdTpQi/woSSCaZmCSOs3E0OmQlKmmYdoxScbgDAlKKHUtGMVOtq1qJjFunMwYocVuEgSVvQOxEeGHDLIcQkDa1bd/nGfXd7Z3772W7rN3z7Of1z8697nPefY8c6T96JznOedRRGBmZrac1qgPwMzMVjcHhZmZ9eSgMDOznhwUZmbWk4PCzMx6clCYmVlPDgozM+vJQWFmZj05KMzMrKf2qA9gGN72trfF9u3bR30YZmaN8vjjj389Isb71SsiKLZv387MzMyoD8PMrFEkfXWQer71ZGZmPTkozMysJweFmZn15KAwM7OeHBRmZtbTQEEhaZekY5JmJd3a5fv1kh5M3x+WtL3ju9tS+TFJV3eU75f0kqQnlrT1Vkl/JOkr6c+3nHv3zMzsfPUNCkkVcDdwDTAJXC9pckm1G4GTEbETuAu4M+07CewBLgN2Afek9gB+L5UtdSvw2YiYAD6bPpuZ2YgM8hzF5cBsRDwNIOkAsBt4sqPObuBX0/ZB4GOSlMoPRMQbwDOSZlN7X4iIz3deeSxp64q0fT/wv4GPDNyjb8Onv3ScZ078bY6mbY1rVy32XL6Vt1/0HaM+FLPzNkhQbAae7fh8HHj/cnUiYk7SKWBjKn90yb6b+/y8TRHxfNp+AdjUrZKkvcBegG3btvXvRRf//c+f53PHXjqnfc2Ws7AM/YXr2/zMD+wY7cGYDcGqfjI7IkJSLPPdvcC9AFNTU13r9LP/p953Hkdn1t1rp+eYvP1hzsyfHfWhmA3FIIPZzwFbOz5vSWVd60hqAxuAlwfcd6kXJb0ztfVOwP/lt0apWgJg7uw5/f/FbNUZJCiOABOSdkhaRz04Pb2kzjRwQ9q+FngkIiKV70mzonYAE8BjfX5eZ1s3AH84wDGarRrtVv3Pam7eQWFl6BsUETEH3AI8DDwFPBQRRyXdIelHU7X7gI1psPoXSTOVIuIo8BD1wPdngJsjYh5A0ieBLwCXSjou6cbU1q8BV0n6CvBD6bNZY1QtIcH8Wd96sjIMNEYREYeAQ0vKbu/Yfh24bpl99wH7upRfv0z9l4ErBzkus9Wq3RJnfOvJCuEns80yqFpi3kFhhXBQmGUw1mp51pMVw0FhlkFV+YrCyuGgMMug3Wp5eqwVw0FhlkG7JeZ868kK4aAwy6BqyVcUVgwHhVkGYx6jsII4KMwyqFryk9lWDAeFWQb1YLbHKKwMDgqzDNqVryisHA4KswzaHsy2gjgozDJoVy0PZlsxHBRmGVQt+RUeVgwHhVkGbb8U0ArioDDLoF35FR5WDgeFWQb1YLZvPVkZHBRmGbT9wJ0VxEFhlkG78vRYK4eDwiyDquXpsVYOB4VZBmMeo7CCOCjMMvBLAa0kDgqzDDxGYSVxUJhl0G61vMKdFcNBYZaBV7izkjgozDLwCndWEgeFWQZVq+XBbCuGg8IsA7/Cw0rioDDLoF2JswFnffvJCuCgMMug3RKAB7StCA4KswyqVv1PywPaVgIHhVkGY1V9RXHG4xRWgIGCQtIuScckzUq6tcv36yU9mL4/LGl7x3e3pfJjkq7u16akKyV9UdL/kfRnknaeXxfNVl6Vbj3Ne+aTFaBvUEiqgLuBa4BJ4HpJk0uq3QicjIidwF3AnWnfSWAPcBmwC7hHUtWnzY8DPx4R3wf8N+DfnV8XzVZeu6r/aXmMwkowyBXF5cBsRDwdEaeBA8DuJXV2A/en7YPAlZKUyg9ExBsR8Qwwm9rr1WYAfy9tbwD++ty6ZjY63xrM9q0na772AHU2A892fD4OvH+5OhExJ+kUsDGVP7pk381pe7k2bwIOSfo74JvAB7odlKS9wF6Abdu2DdANs5WzcOvJD91ZCVbjYPaHgR+OiC3A7wIf7VYpIu6NiKmImBofH1/RAzTrZ2Ew27OerASDBMVzwNaOz1tSWdc6ktrUt4xe7rFv13JJ48D3RsThVP4g8A8H6onZKrIwPda3nqwEgwTFEWBC0g5J66gHp6eX1JkGbkjb1wKPRESk8j1pVtQOYAJ4rEebJ4ENki5JbV0FPHXu3TMbjTE/cGcF6TtGkcYcbgEeBipgf0QclXQHMBMR08B9wAOSZoFXqH/xk+o9BDwJzAE3R8Q8QLc2U/nPAr8v6Sx1cPzMUHtstgI8RmElGWQwm4g4BBxaUnZ7x/brwHXL7LsP2DdIm6n808CnBzkus9WqXfmKwsqxGgezzRqvvfgKD49RWPM5KMwyWHiO4oxvPVkBHBRmGSy+wsO3nqwADgqzDBZe4XFm3reerPkcFGYZtH1FYQVxUJhl4FlPVhIHhVkGC7Oe/ByFlcBBYZZB5bfHWkEcFGYZ+KWAVhIHhVkGfoWHlcRBYZbB4hiFryisAA4Kswy+NevJYxTWfA4KswzavvVkBXFQmGWw8GS2B7OtBA4KswwWXwroW09WAAeFWQaLLwX0rScrgIPCLIO2l0K1gjgozDKQRNWSZz1ZERwUZpm0W/IVhRXBQWGWSbslT4+1IjgozDKpWvL0WCuCg8Isk7Gq5TEKK4KDwiyTyreerBAOCrNMPJhtpXBQmGXSrlrMzfvWkzWfg8IsE19RWCkcFGaZtCvPerIyOCjMMqlaLc54MNsK4KAwy6TdEvOeHmsFcFCYZdKuPEZhZRgoKCTtknRM0qykW7t8v17Sg+n7w5K2d3x3Wyo/Junqfm2qtk/SlyU9Jennz6+LZqPhV3hYKdr9KkiqgLuBq4DjwBFJ0xHxZEe1G4GTEbFT0h7gTuBDkiaBPcBlwLuAP5Z0SdpnuTZ/CtgKvDsizkp6+zA6arbS/AoPK8UgVxSXA7MR8XREnAYOALuX1NkN3J+2DwJXSlIqPxARb0TEM8Bsaq9Xmz8H3BERZwEi4qVz757Z6IxVLa9wZ0UYJCg2A892fD6eyrrWiYg54BSwsce+vdr8LuqrkRlJ/0vSxGBdMVtdfEVhpViNg9nrgdcjYgr4bWB/t0qS9qYwmTlx4sSKHqDZINqtlscorAiDBMVz1GMGC7aksq51JLWBDcDLPfbt1eZx4FNp+9PA93Q7qIi4NyKmImJqfHx8gG6Yray2V7izQgwSFEeACUk7JK2jHpyeXlJnGrghbV8LPBIRkcr3pFlRO4AJ4LE+bf4B8E/T9j8BvnxuXTMbrcrTY60QfWc9RcScpFuAh4EK2B8RRyXdAcxExDRwH/CApFngFepf/KR6DwFPAnPAzRExD9CtzfQjfw34hKQPA68CNw2vu2YrZ8xjFFaIvkEBEBGHgENLym7v2H4duG6ZffcB+wZpM5V/A/iRQY7LbDWrPEZhhViNg9lmRRirPEZhZXBQmGXiFe6sFA4Ks0y8HoWVwkFhlkm7ankw24rgoDDLpN0SZ7wUqhXAQWGWiV/hYaVwUJhl0q5azJ0N6mdPzZrLQWGWSbslAF9VWOM5KMwyaVd1UHjmkzWdg8Isk4UrCgeFNZ2DwiyTqlX/85r3Q3fWcA4Ks0zGFm89eYqsNZuDwiyTyreerBAOCrNMPEZhpXBQmGXSTmMUc3462xrOQWGWiafHWikcFGaZLFxR+IE7azoHhVkmC4PZfjGgNZ2DwiwTv8LDSuGgMMvEYxRWCgeFWSbfmvXkoLBmc1CYZdL2k9lWCAeFWSaLD9z5isIazkFhlknlwWwrhIPCLJOxKo1ROCis4RwUZpksvhTQz1FYwzkozDLxSwGtFA4Ks0zai7eefEVhzeagMMvEs56sFA4Ks0wWnqPwrCdrOgeFWSaLLwV0UFjDDRQUknZJOiZpVtKtXb5fL+nB9P1hSds7vrstlR+TdPW30eZvSnr13LplNnqLrxn3rCdruL5BIakC7gauASaB6yVNLql2I3AyInYCdwF3pn0ngT3AZcAu4B5JVb82JU0BbznPvpmNlF8KaKUY5IricmA2Ip6OiNPAAWD3kjq7gfvT9kHgSklK5Qci4o2IeAaYTe0t22YKkf8E/PL5dc1stDw91koxSFBsBp7t+Hw8lXWtExFzwClgY499e7V5CzAdEc8P1gWz1cmv8LBStEd9AJ0kvQu4DrhigLp7gb0A27Zty3tgZudgLI1ReIU7a7pBriieA7Z2fN6SyrrWkdQGNgAv99h3ufL3AjuBWUl/BVwgabbbQUXEvRExFRFT4+PjA3TDbGW1WkLyFYU13yBBcQSYkLRD0jrqwenpJXWmgRvS9rXAIxERqXxPmhW1A5gAHluuzYj4nxHxjojYHhHbgdfSALlZI421Wh6jsMbre+spIuYk3QI8DFTA/og4KukOYCYipoH7gAfS//5fof7FT6r3EPAkMAfcHBHzAN3aHH73zEaraskvBbTGG2iMIiIOAYeWlN3esf069dhCt333AfsGabNLnQsHOT6z1ardkq8orPH8ZLZZRu1KHqOwxnNQmGVUtVqc8UsBreEcFGYZjVVi3q8Zt4ZzUJhlVA9m+4rCms1BYZaRB7OtBA4Ks4zaVcuD2dZ4DgqzjNot+RUe1ngOCrOMqpanx1rzOSjMMmpXLa9wZ43noDDLqN3y9FhrPgeFWUZtT4+1AjgozDJqV54ea83noDDLqPJrxq0ADgqzjMY8RmEFcFCYZeRXeFgJHBRmGXmMwkrgoDDLqN1qeYU7azwHhVlGfimglcBBYZaRV7izEjgozDLyCndWAgeFWUZ+hYeVwEFhlpFnPVkJHBRmGfldT1YCB4VZRl7hzkrgoDDLqN0SZzxGYQ3noDDLqGqJCDjrqwprMAeFWUZjVf1PzAPa1mQOCrOMqpYAmPPtJ2swB4VZRu3FoPAVhTWXg8Iso8Wg8BRZazAHhVlG1eIYhW89WXMNFBSSdkk6JmlW0q1dvl8v6cH0/WFJ2zu+uy2VH5N0db82JX0ilT8hab+ksfProtnojKUrCj9LYU3WNygkVcDdwDXAJHC9pMkl1W4ETkbETuAu4M607ySwB7gM2AXcI6nq0+YngHcD7wHeBNx0Xj00G6HKt56sAINcUVwOzEbE0xFxGjgA7F5SZzdwf9o+CFwpSan8QES8ERHPALOpvWXbjIhDkQCPAVvOr4tmo9OuPJhtzTdIUGwGnu34fDyVda0TEXPAKWBjj337tpluOf0k8JkBjtFsVWq36n9ifoOsNdlqHsy+B/h8RPxpty8l7ZU0I2nmxIkTK3xoZoNZmPXkNSmsyQYJiueArR2ft6SyrnUktYENwMs99u3ZpqRfAcaBX1zuoCLi3oiYioip8fHxAbphtvIqD2ZbAQYJiiPAhKQdktZRD05PL6kzDdyQtq8FHkljDNPAnjQragcwQT3usGybkm4Crgaujwhfr1ujLbzC48y8/ypbc7X7VYiIOUm3AA8DFbA/Io5KugOYiYhp4D7gAUmzwCvUv/hJ9R4CngTmgJsjYh6gW5vpR/4W8FXgC/V4OJ+KiDuG1mOzFeQrCitB36CAeiYScGhJ2e0d268D1y2z7z5g3yBtpvKBjsmsCTzryUqwmgezzRpvYdaTn6OwJnNQmGXkt8daCRwUZhmNVR6jsOZzUJhlVPk5CiuAg8Iso4Xpsb6isCZzUJhl5DEKK4GDwiwjL1xkJXBQmGXU9q0nK4CDwiyjxZcC+taTNZiDwiwjv8LDSuCgMMtorLXwUkAHhTWXg8Iso2rxgTvferLmclCYZbQ468m3nqzBHBRmGXl6rJXAr/Q2y2hhMPuBR7/Kw0dfGPHR5Pc9Wy7mP/7z94z6MGzIHBRmGUni5674Lr7y4t+M+lCy+9orr3HgyNe4/YOTvGldNerDsSFyUJhl9pFd7x71IayIzzzxPP/qv36R2Zde5T1bNoz6cGyIPEZhZkMxsekiAI6tgauntcZBYWZD8Z1vvYB17RZfdlAUx0FhZkPRrlrsHL+QYy84KErjoDCzobn0HRetiYH7tcZBYWZDc8mmi/jrU6/zzdfPjPpQbIgcFGY2NJe+40IAX1UUxkFhZkMz8fY08+mFV0d8JDZMDgozG5rNF7+JN6+rPPOpMA4KMxuaVktMbLrIQVEYB4WZDdWlDoriOCjMbKguecdFfP3V03z91TdGfSg2JA4KMxuqSzbVM598VVEOB4WZDdWl6Z1PX/YT2sVwUJjZUI1ftJ6LLxjj2IueIluKgYJC0i5JxyTNSrq1y/frJT2Yvj8saXvHd7el8mOSru7XpqQdqY3Z1Oa68+uima0kSVyyya/yKEnf9SgkVcDdwFXAceCIpOmIeLKj2o3AyYjYKWkPcCfwIUmTwB7gMuBdwB9LuiTts1ybdwJ3RcQBSb+V2v74MDprZivj0k0X8cnHvsZVH/2TUR9K8e674X1s23hB1p8xyMJFlwOzEfE0gKQDwG6gMyh2A7+atg8CH5OkVH4gIt4AnpE0m9qjW5uSngJ+EPixVOf+1K6DwqxBPvS+rZx87TRnw2uF57aunX8EYZCg2Aw82/H5OPD+5epExJykU8DGVP7okn03p+1ubW4EvhERc13qm1lDfPfmDXzsx75/1IdhQ9LYwWxJeyXNSJo5ceLEqA/HzKxYgwTFc8DWjs9bUlnXOpLawAbg5R77Llf+MnBxamO5nwVARNwbEVMRMTU+Pj5AN8zM7FwMEhRHgIk0G2kd9eD09JI608ANafta4JGIiFS+J82K2gFMAI8t12ba53OpDVKbf3ju3TMzs/PVd4wijTncAjwMVMD+iDgq6Q5gJiKmgfuAB9Jg9SvUv/hJ9R6iHvieA26OiHmAbm2mH/kR4ICk/wB8KbVtZmYjoihgVsLU1FTMzMyM+jDMzBpF0uMRMdWvXmMHs83MbGU4KMzMrCcHhZmZ9VTEGIWkE8BXz3H3twFfH+LhNMVa7Pda7DOszX67z4P5zojo+3xBEUFxPiTNDDKYU5q12O+12GdYm/12n4fLt57MzKwnB4WZmfXkoIB7R30AI7IW+70W+wxrs9/u8xCt+TEKMzPrzVcUZmbW05oOin5LvJZA0lZJn5P0pKSjkn4hlb9V0h9J+kr68y2jPtZhk1RJ+pKk/5E+F7/MrqSLJR2U9H8lPSXpH5R+riV9OP3dfkLSJyV9R4nnWtJ+SS9JeqKjrOu5Ve03U///QtJ5LQ6yZoOiY4nXa4BJ4Pq0dGtp5oB/ExGTwAeAm1M/bwU+GxETwGfT59L8AvBUx+eFZXZ3Aiepl9ktzW8An4mIdwPfS93/Ys+1pM3AzwNTEfHd1C8ZXViOubRz/XvAriVly53ba6jf1j0B7OU8Vwlds0FBxxKvEXEaWFjitSgR8XxEfDFt/w31L47N1H29P1W7H/hnoznCPCRtAX4E+J30WdTL7B5MVUrs8wbgH5PeuBwRpyPiGxR+rqnfgv2mtI7NBcDzFHiuI+Lz1G/n7rTcud0N/JeoPUq9zs87z/Vnr+Wg6LbEa9HLrkraDrwXOAxsiojn01cvAJtGdFi5/Drwy8DZ9HktLLO7AzgB/G665fY7kt5Mwec6Ip4D/jPwNeqAOAU8TvnnesFy53aov9/WclCsKZIuBH4f+NcR8c3O79KCUcVMf5P0QeCliHh81MeywtrA9wMfj4j3An/LkttMBZ7rt1D/73kH8C7gzfz/t2fWhJzndi0HxSBLvBZB0hh1SHwiIj6Vil9cuBRNf740quPL4B8BPyrpr6hvKf4g9b37gZbZbbDjwPGIOJw+H6QOjpLP9Q8Bz0TEiYg4A3yK+vyXfq4XLHduh/r7bS0HxSBLvDZeujd/H/BURHy046vO5WuLWnI2Im6LiC0RsZ36vD4SET9O4cvsRsQLwLOSLk1FV1KvLlnsuaa+5fQBSRekv+sLfS76XHdY7txOA/8yzX76AHCq4xbVt21NP3An6Yep72UvLMe6b8SHNHSSfgD4U+Av+db9+n9LPU7xELCN+s27/yIilg6UNZ6kK4BfiogPSvr71FcYb6VeZvcnIuKNUR7fsEn6PuoB/HXA08BPU/+HsNhzLenfAx+inuH3JeAm6vvxRZ1rSZ8ErqB+S+yLwK8Af0CXc5tC82PUt+FeA346Is55GdA1HRRmZtbfWr71ZGZmA3BQmJlZTw4KMzPryUFhZmY9OSjMzKwnB4WZmfXkoDAzs54cFGZm1tP/AxGebOVAI4/sAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[50, 70], gamma=0.1)\n",
    "\n",
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "    scheduler.step()\n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### (3)　公式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x118d4fc10>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGU1JREFUeJzt3X2MXNd93vHvszO7s3zZ5TsZhy8iE7KxKRm2pa2sImqbWJFNpY6pIhJMQ7CEQjAD2ELSpG1KozDhCsofAtKqNawaZSzJEltbcpg4Xth0KNuSYUSoKS4lxRIpM15Rb6RlvokiRVl8WfLXP+YsNV7P7F6S836fDzDYO3fOvXsurrQPzzn3nFFEYGZm1tPqCpiZWXtwIJiZGeBAMDOzxIFgZmaAA8HMzBIHgpmZAQ4EMzNLHAhmZgY4EMzMLCm2ugIXYv78+bF8+fJWV8PMrKPs3LnzcEQsmKpcRwXC8uXLGRkZaXU1zMw6iqSXs5Rzl5GZmQEOBDMzSxwIZmYGOBDMzCxxIJiZGZAxECStkbRH0qikDVU+L0l6JH2+XdLytP9qSc+k1z9K+rdZz2lmZs01ZSBIKgD3AjcAq4FPSFo9odjtwNGIWAncA9yd9j8HDEXE+4E1wP+WVMx4TjMza6Is8xCuBkYjYi+ApIeBtcDuijJrgc+n7S3AFyUpIn5RUaYfGP++ziznrJuvPPEir791uhGn7lqSuOmqJSydO73VVTGzJskSCIuBVyve7wM+WKtMRIxJOgbMAw5L+iBwP3AZ8Mn0eZZzAiBpPbAeYNmyZRmq+6u++uQr/PTgiYs6Nq8i4PTZc/znNe9udVXMrEkaPlM5IrYDl0t6D/CgpO9c4PGbgE0AQ0NDMUXxqh790399MYfl2ns/v42TZ862uhpm1kRZBpX3A0sr3i9J+6qWkVQEZgFHKgtExPPACeCKjOe0FioVC5waO9fqaphZE2UJhB3AKkkrJPUB64DhCWWGgdvS9k3AYxER6ZgigKTLgHcDL2U8p7VQqdjDqTMOBLM8mbLLKPX53wFsAwrA/RGxS9KdwEhEDAP3AZsljQKvU/4DD3AtsEHSGeAc8OmIOAxQ7Zx1vja7BKViD6fG3GVklieZxhAiYiuwdcK+jRXbJ4Gbqxy3Gdic9ZzWPvqKPZx2l5FZrnimslVV6vUYglneOBCsqlLBXUZmeeNAsKpKve4yMssbB4JVVR5UdiCY5YkDwarqcyCY5Y4DwaoqFQvuMjLLGQeCVeV5CGb540CwqjyGYJY/DgSryhPTzPLHgWBVeXE7s/xxIFhVpWIPZ88FY2cdCmZ54UCwqvqK5f803Eowyw8HglVVSoHgcQSz/HAgWFWl3gLgFoJZnjgQrKq+wniXkecimOWFA8GqKvW6y8gsbxwIVlWp6C4js7xxIFhVpaK7jMzyxoFgVfmxU7P8cSBYVSUHglnuOBCsqvNjCGccCGZ54UCwqvo8hmCWOw4Eq8ozlc3yx4FgVY3PQ/AYgll+OBCsqlLB8xDM8iZTIEhaI2mPpFFJG6p8XpL0SPp8u6Tlaf/1knZKejb9/FDFMT9I53wmvRbW66Ls0nmmsln+FKcqIKkA3AtcD+wDdkgajojdFcVuB45GxEpJ64C7gY8Dh4E/iIifSboC2AYsrjjulogYqdO1WB15LSOz/MnSQrgaGI2IvRFxGngYWDuhzFrgwbS9BbhOkiLi6Yj4Wdq/C5gmqVSPiltj9fSI3oLcZWSWI1kCYTHwasX7ffzyv/J/qUxEjAHHgHkTyvwh8FREnKrY90DqLvqcJFX75ZLWSxqRNHLo0KEM1bV6KRUL7jIyy5GmDCpLupxyN9IfVey+JSLeC/zL9PpktWMjYlNEDEXE0IIFCxpfWTuvVOxxl5FZjmQJhP3A0or3S9K+qmUkFYFZwJH0fgnwDeDWiHhh/ICI2J9+vgl8lXLXlLWRUrHHM5XNciRLIOwAVklaIakPWAcMTygzDNyWtm8CHouIkDQb+DawISKeGC8sqShpftruBT4KPHdpl2L11lfs4fRZB4JZXkwZCGlM4A7KTwg9D3w9InZJulPSx1Kx+4B5kkaBPwPGH029A1gJbJzweGkJ2Cbpx8AzlFsYf1XPC7NLVyoW3EIwy5EpHzsFiIitwNYJ+zZWbJ8Ebq5y3F3AXTVOe1X2alorlHo9hmCWJ56pbDX1FXr82KlZjjgQrKZSb48fOzXLEQeC1VQqFtxCMMsRB4LVVO4y8hiCWV44EKwmdxmZ5YsDwWoqz1R2IJjlhQPBaupzIJjligPBavLidmb54kCwmry4nVm+OBCsplKxwJmzwblz0eqqmFkTOBCspr5i+hpNL3BnlgsOBKuplALBC9yZ5YMDwWoq9fp7lc3yxIFgNfUVxgPBLQSzPHAgWE2l3gLgQDDLCweC1XR+DMFdRma54ECwmvqK7jIyyxMHgtU03kLwbGWzfHAgWE2loscQzPLEgWA1vTMPwWMIZnngQLCaSp6pbJYrDgSr6XyXkWcqm+WCA8FqememsgPBLA8cCFbT+Ezl056HYJYLDgSryS0Es3zJFAiS1kjaI2lU0oYqn5ckPZI+3y5pedp/vaSdkp5NPz9UccxVaf+opC9IUr0uyurDaxmZ5cuUgSCpANwL3ACsBj4hafWEYrcDRyNiJXAPcHfafxj4g4h4L3AbsLnimC8BnwJWpdeaS7gOa4BioYdCj7x0hVlOZGkhXA2MRsTeiDgNPAysnVBmLfBg2t4CXCdJEfF0RPws7d8FTEutiXcBgxHxo4gI4CHgxku+Gqu7UrHHM5XNciJLICwGXq14vy/tq1omIsaAY8C8CWX+EHgqIk6l8vumOCcAktZLGpE0cujQoQzVtXoqf6+yA8EsD5oyqCzpcsrdSH90ocdGxKaIGIqIoQULFtS/cjapvmKP5yGY5USWQNgPLK14vyTtq1pGUhGYBRxJ75cA3wBujYgXKsovmeKc1gZKxYJnKpvlRJZA2AGskrRCUh+wDhieUGaY8qAxwE3AYxERkmYD3wY2RMQT44Uj4jXguKRr0tNFtwLfvMRrsQYodxl5UNksD6YMhDQmcAewDXge+HpE7JJ0p6SPpWL3AfMkjQJ/Bow/mnoHsBLYKOmZ9FqYPvs08GVgFHgB+E69Lsrqx11GZvlRzFIoIrYCWyfs21ixfRK4ucpxdwF31TjnCHDFhVTWmq9U7HGXkVlOeKayTapULLiFYJYTDgSbVKnXYwhmeeFAsEn1FTwPwSwvHAg2qVJvwTOVzXLCgWCT8kxls/xwINik+jwPwSw3HAg2KbcQzPLDgWCTKhULDgSznHAg2KT60vLX5VXKzaybORBsUqVi+l5lz1Y263oOBJvUeCC428is+zkQbFLnA8HLV5h1PQeCTapULADuMjLLAweCTarUO95C8FwEs27nQLBJeQzBLD8cCDapvvGnjBwIZl3PgWCTGh9DcAvBrPs5EGxS73QZeQzBrNs5EGxSfX7s1Cw3HAg2KT92apYfDgSblLuMzPLDgWCTcpeRWX44EGxSXtzOLD8cCDapUm967NQtBLOu50CwSfUVPIZglheZAkHSGkl7JI1K2lDl85KkR9Ln2yUtT/vnSXpc0glJX5xwzA/SOZ9Jr4X1uCCrr96CkDxT2SwPilMVkFQA7gWuB/YBOyQNR8TuimK3A0cjYqWkdcDdwMeBk8DngCvSa6JbImLkEq/BGkiSv1fZLCemDATgamA0IvYCSHoYWAtUBsJa4PNpewvwRUmKiLeAf5C0sn5VtmYrFQu8ePgtnhg9nPmYZXOns3Tu9AbWyszqLUsgLAZerXi/D/hgrTIRMSbpGDAPmOovyAOSzgJ/A9wVVb64V9J6YD3AsmXLMlTX6m3ujD4e3X2AR3cfyHzMsrnT+eGf/24Da2Vm9ZYlEBrllojYL2mAciB8EnhoYqGI2ARsAhgaGvI3vbfA1z51Da+8/ovM5f/Pj17m0d0/b2CNzKwRsgTCfmBpxfslaV+1MvskFYFZwJHJThoR+9PPNyV9lXLX1K8EgrXer83q59dm9Wcuv33vEYb/8Rynx86dn9hmZu0vy/+tO4BVklZI6gPWAcMTygwDt6Xtm4DHqnX/jJNUlDQ/bfcCHwWeu9DKW3sa6C//O+PNk2daXBMzuxBTthDSmMAdwDagANwfEbsk3QmMRMQwcB+wWdIo8Drl0ABA0kvAINAn6Ubgw8DLwLYUBgXge8Bf1fXKrGUG+nsBePPkGPNmllpcGzPLKtMYQkRsBbZO2LexYvskcHONY5fXOO1V2aponWZwWjkQjruFYNZR3MFrdfdOl9FYi2tiZhfCgWB1N3i+y8gtBLNO4kCwuhtvIRx/2y0Es07iQLC68xiCWWdyIFjdzSylFoLHEMw6igPB6q7QI2aWih5DMOswDgRriMH+op8yMuswDgRriIH+Xo6/7RaCWSdxIFhDDE5zC8Gs0zgQrCEG+nv9lJFZh3EgWEMMeAzBrOM4EKwhBvt7/ZSRWYdxIFhDDPQXOX5yjElWQTezNuNAsIYYnNbL2XPB22fOtroqZpaRA8EawusZmXUeB4I1xIBXPDXrOA4Ea4jBfq9nZNZpHAjWEOMtBM9FMOscDgRriFnT/K1pZp3GgWAN4TEEs87jQLCGGP8aTT9lZNY5HAjWEP29PRR75BaCWQdxIFhDSEqzlR0IZp3CgWANMzit14PKZh3EgWAN4xVPzTpLpkCQtEbSHkmjkjZU+bwk6ZH0+XZJy9P+eZIel3RC0hcnHHOVpGfTMV+QpHpckLWPQX9rmllHmTIQJBWAe4EbgNXAJyStnlDsduBoRKwE7gHuTvtPAp8D/mOVU38J+BSwKr3WXMwFWPtyC8Gss2RpIVwNjEbE3og4DTwMrJ1QZi3wYNreAlwnSRHxVkT8A+VgOE/Su4DBiPhRlNdHfgi48VIuxNqPvzXNrLNkCYTFwKsV7/elfVXLRMQYcAyYN8U5901xTutw5S/JcQvBrFO0/aCypPWSRiSNHDp0qNXVsQsw0F/kxKkxzp7zl+SYdYIsgbAfWFrxfknaV7WMpCIwCzgyxTmXTHFOACJiU0QMRcTQggULMlTX2sXgtPJs5RNuJZh1hCyBsANYJWmFpD5gHTA8ocwwcFvavgl4LCb57sSIeA04Luma9HTRrcA3L7j21tbOf0mOxxHMOkJxqgIRMSbpDmAbUADuj4hdku4ERiJiGLgP2CxpFHidcmgAIOklYBDok3Qj8OGI2A18GvgKMA34TnpZFxl0IJh1lCkDASAitgJbJ+zbWLF9Eri5xrHLa+wfAa7IWlHrPIPnVzx1l5FZJ2j7QWXrXAMOBLOO4kCwhhlMX5Lj2cpmncGBYA3jL8kx6ywOBGuYd54ycpeRWSdwIFjD9BZ6mNZbcAvBrEM4EKyhvMCdWedwIFhDDU7zAndmncKBYA3lFoJZ53AgWEMN9vd6UNmsQzgQrKEG+ou86XkIZh0h09IVZhdroL+XN94+w0uH38p8zLS+AosG+xtYKzOrxoFgDTV/Zh+vv3Wa3/nLH1zQcd/8zG/zvqWzG1MpM6vKgWANdfu1K1i5cCbnaq+G/kuOnDjNXd9+nhcOnXAgmDWZA8Eaavb0Pta+P/u3o751aoy7vv08B46famCtzKwaDypbW5lRKjKzVOTA8ZOtropZ7jgQrO0sHCxx6E23EMyazYFgbWfhQMktBLMWcCBY21k02M+BNx0IZs3mQLC2s2iwn4PHTxEZn0wys/pwIFjbWThQ4tTYOY6/7SUvzJrJgWBtZ2GapXzQ3UZmTeVAsLazaKAE4LkIZk3mQLC2M95C8JNGZs3lQLC2szC1EA56LoJZUzkQrO3MKBUZ8Gxls6bLFAiS1kjaI2lU0oYqn5ckPZI+3y5pecVnn03790j6SMX+lyQ9K+kZSSP1uBjrHgsGSx5UNmuyKRe3k1QA7gWuB/YBOyQNR8TuimK3A0cjYqWkdcDdwMclrQbWAZcDvw58T9I/i4iz6bjfjYjDdbwe6xKLBspzEcysebK0EK4GRiNib0ScBh4G1k4osxZ4MG1vAa6TpLT/4Yg4FREvAqPpfGaTWjhY8mxlsybLEgiLgVcr3u9L+6qWiYgx4Bgwb4pjA3hU0k5J6y+86tbNFg32c8Czlc2aqpXfh3BtROyXtBD4rqSfRMQPJxZKYbEeYNmyZc2uo7XIwoESp9Ns5VnTe1tdHbNcyNJC2A8srXi/JO2rWkZSEZgFHJns2IgY/3kQ+AY1upIiYlNEDEXE0IIFCzJU17rB+bkI7jYya5osgbADWCVphaQ+yoPEwxPKDAO3pe2bgMei3NYfBtalp5BWAKuAJyXNkDQAIGkG8GHguUu/HOsW47OVPbBs1jxTdhlFxJikO4BtQAG4PyJ2SboTGImIYeA+YLOkUeB1yqFBKvd1YDcwBnwmIs5KWgR8ozzuTBH4akT8fQOuzzrUIs9WNmu6TGMIEbEV2Dph38aK7ZPAzTWO/QvgLybs2wu870Ira/mxcDCtZ+QuI7Om8Uxla0vT+8qzld1lZNY8DgRrW56tbNZcDgRrW4sG+r0EtlkTORCsbS1yC8GsqRwI1rYWerayWVM5EKxtjc9WPvb2mVZXxSwXHAjWthad/25ljyOYNUMr1zIym9T4N6c98MRLLJs7PfNxv/eehaxaNNCoapl1LQeCta3fXDiTmaUiX3vylQs67skXj/DAv/Mq62YXyoFgbWv+zBJPb7yes+eyDyp/7u+e49HdBzh3LujpUQNrZ9Z9PIZgba230EN/byHz65+vmMuxt8+w9/CJVlfdrOM4EKyrXHXZHAB2vny0xTUx6zwOBOsqvzF/BrOn9zoQzC6CA8G6iiSuXDaHp155o9VVMes4DgTrOlddNofRgyd44xenW10Vs47iQLCu84FlswF42q0EswviQLCu874lsyn0iKde8TiC2YVwIFjXmVEq8p53DXhg2ewCORCsK121bA7PvPoGY2fPtboqZh3DgWBd6crL5vCL02fZc+DNVlfFrGM4EKwrXbmsPEHtKXcbmWXmtYysKy2ZM42FAyX+23f/iYf+38utro7ZJfvWH19LqVho6O9wIFhXksR/+shv8fieg62uilldiMYv1uhAsK5189BSbh5a2upqmHUMjyGYmRmQMRAkrZG0R9KopA1VPi9JeiR9vl3S8orPPpv275H0kaznNDOz5poyECQVgHuBG4DVwCckrZ5Q7HbgaESsBO4B7k7HrgbWAZcDa4D/JamQ8ZxmZtZEWVoIVwOjEbE3Ik4DDwNrJ5RZCzyYtrcA10lS2v9wRJyKiBeB0XS+LOc0M7MmyhIIi4FXK97vS/uqlomIMeAYMG+SY7Oc08zMmqjtB5UlrZc0Imnk0KFDra6OmVnXyhII+4HKZ/eWpH1Vy0gqArOAI5Mcm+WcAETEpogYioihBQsWZKiumZldjCyBsANYJWmFpD7Kg8TDE8oMA7el7ZuAxyIi0v516SmkFcAq4MmM5zQzsyaacmJaRIxJugPYBhSA+yNil6Q7gZGIGAbuAzZLGgVep/wHnlTu68BuYAz4TEScBah2zqnqsnPnzsOSLnYdgvnA4Ys8tlPl8Zohn9edx2uGfF73xVzzZVkKqfwP+e4naSQihlpdj2bK4zVDPq87j9cM+bzuRl5z2w8qm5lZczgQzMwMyFcgbGp1BVogj9cM+bzuPF4z5PO6G3bNuRlDMDOzyeWphWBmZpPo+kDIy6qqkpZKelzSbkm7JP1J2j9X0ncl/TT9nNPqutZbWjDxaUnfSu9XpFV3R9MqvH2trmO9SZotaYukn0h6XtK/6PZ7LelP03/bz0n6mqT+brzXku6XdFDScxX7qt5blX0hXf+PJV15Kb+7qwMhZ6uqjgH/ISJWA9cAn0nXugH4fkSsAr6f3nebPwGer3h/N3BPWn33KOXVeLvN/wT+PiLeDbyP8vV37b2WtBj4Y2AoIq6gPH9pHd15r79CeXXoSrXu7Q2UJ/yuAtYDX7qUX9zVgUCOVlWNiNci4qm0/SblPxCL+eWVaB8EbmxNDRtD0hLg3wBfTu8FfIjyqrvQndc8C/hXlCeEEhGnI+INuvxeU55IOy0tjzMdeI0uvNcR8UPKE3wr1bq3a4GHouxHwGxJ77rY393tgZDLVVXTFxR9ANgOLIqI19JHPwcWtahajfI/gD8HzqX384A30qq70J33fAVwCHggdZV9WdIMuvheR8R+4C+BVygHwTFgJ91/r8fVurd1/RvX7YGQO5JmAn8D/PuIOF75WVpfqmseK5P0UeBgROxsdV2arAhcCXwpIj4AvMWE7qEuvNdzKP9reAXw68AMfrVbJRcaeW+7PRAyr6raDST1Ug6D/xsRf5t2HxhvQqafB1tVvwb4beBjkl6i3B34Icp967NTtwJ05z3fB+yLiO3p/RbKAdHN9/r3gBcj4lBEnAH+lvL97/Z7Pa7Wva3r37huD4TcrKqa+s7vA56PiP9e8VHlSrS3Ad9sdt0aJSI+GxFLImI55Xv7WETcAjxOedVd6LJrBoiInwOvSvqttOs6ygtIdu29ptxVdI2k6em/9fFr7up7XaHWvR0Gbk1PG10DHKvoWrpwEdHVL+D3gX8CXgD+S6vr08DrvJZyM/LHwDPp9fuU+9S/D/wU+B4wt9V1bdD1/w7wrbT9G5SXWR8F/hootbp+Dbje9wMj6X7/HTCn2+818F+BnwDPAZuBUjfea+BrlMdJzlBuDd5e694Covwk5QvAs5Sfwrro3+2ZymZmBnR/l5GZmWXkQDAzM8CBYGZmiQPBzMwAB4KZmSUOBDMzAxwIZmaWOBDMzAyA/w8gmEJcFxl7OgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def adjust_learning_rate(optimizer, epoch):\n",
    "    \"\"\" \"\"\"\n",
    "    lr = 30e-3\n",
    "    if epoch > 25:\n",
    "        lr = 15e-3\n",
    "    if epoch > 30:\n",
    "        lr = 7.5e-3\n",
    "    if epoch > 35:\n",
    "        lr = 3e-3\n",
    "    if epoch > 40:\n",
    "        lr = 1e-3\n",
    "        \n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    adjust_learning_rate(optimizer, epoch)\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "plt.plot(x,y) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.　指数方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr=lr∗gamma∗∗epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x118e3b190>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHStJREFUeJzt3Xl0XPWZ5vHvW1XaJUuytdiWJW94wTYGG4NJIB3WCRACpLMBId3JkDCZNDNJh5MZMplJd9OTnmbSnU7SIXScwJBkEgghm09iloRlCJuxiAHvWJaNLeNFlhfZkrXWO39U2RRCtspWSVd16/kc16m7/FT3vefKT1397mbujoiIhEsk6AJERCTzFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhGJBLbiqqsqnTZsW1OJFRLLSyy+/vM/dq4dqF1i4T5s2jcbGxqAWLyKSlczsjXTaqVtGRCSEFO4iIiGkcBcRCSGFu4hICCncRURCaMhwN7P7zGyvma09wXwzs2+bWZOZvWZmizNfpoiInIp09tzvB648yfyrgFnJ163APcMvS0REhmPIcHf3Z4D9J2lyHfAjT3gRqDCzSZkqcKDGbfu569GN6PGAIiInlok+9zpgR8p4S3LaO5jZrWbWaGaNra2tp7WwtTsPcc/TW9h3pOe0fl5EJBeM6gFVd1/m7kvcfUl19ZBXzw5qRnUpAFtaj2SyNBGRUMlEuO8E6lPGpySnjYiZNQp3EZGhZCLclwN/kTxr5gLgkLvvysDnDmrSuEKK8qI0t3aM1CJERLLekDcOM7MHgIuBKjNrAf4GyANw938DVgBXA01AJ/CpkSoWIBIxZlSXaM9dROQkhgx3d79xiPkO/FXGKkrDzOpSVu84MJqLFBHJKll5herM6lJaDhylq7c/6FJERMakrAz3GdUluMPWfep3FxEZTFaG+0ydDikiclJZGe7Tq0owgy17tecuIjKYrAz3ovwodRVFNO/TnruIyGCyMtwh0TWjbhkRkcFlbbjPqC5hy94O4nHdQExEZKCsDfeZ1aUc7e1nd3tX0KWIiIw5WR3uoDNmREQGk73hXlMCwJa9CncRkYGyNtyrSwsoK4zRrAuZRETeIWvD3cx0xoyIyAlkbbjDW2fMiIjI22V1uM+sLmV3exeHu3qDLkVEZEzJ6nCflXwqU5MOqoqIvE1Wh/uciWUAvL7ncMCViIiMLVkd7vWVxRTmRdi0W3vuIiKpsjrcIxFjdm2Z9txFRAbI6nAHmF1bxiaFu4jI22R9uM+dWEbr4W72d/QEXYqIyJiR9eE+uzZxUHXTbu29i4gck/XhrjNmRETeKevDvaasgPKiPPW7i4ikyPpwNzPm1JbxurplRESOy/pwh0TXzKY9h3HXU5lERCAk4T57YhmHu/rYdUhPZRIRgZCE+5xjZ8yo311EBAhJuM+uTdxATP3uIiIJoQj3iuJ8ascV6Fx3EZGkUIQ7wJyJ49QtIyKSlFa4m9mVZrbJzJrM7I5B5jeY2VNmttrMXjOzqzNf6snNqS1l894j9Md1xoyIyJDhbmZR4G7gKmAecKOZzRvQ7L8DD7n7IuAG4LuZLnQos2vL6OmLs61Nj90TEUlnz/18oMndm929B3gQuG5AGwfGJYfLgTczV2J6zpyUWPz6N9tHe9EiImNOOuFeB+xIGW9JTkv1t8DNZtYCrAD+U0aqOwWza8vIixrrFO4iIhk7oHojcL+7TwGuBn5sZu/4bDO71cwazayxtbU1Q4tOyI9FmFVTxro3D2X0c0VEslE64b4TqE8Zn5KcluoW4CEAd38BKASqBn6Quy9z9yXuvqS6uvr0Kj6J+ZPHsf7Ndt2GQERyXjrhvgqYZWbTzSyfxAHT5QPabAcuAzCzM0mEe2Z3zdMwf/I42jp62NPePdqLFhEZU4YMd3fvA24DHgM2kDgrZp2Z3Wlm1yab3Q58xsxeBR4APukB7D4vqCsHYO1Odc2ISG6LpdPI3VeQOFCaOu2rKcPrgQszW9qpO3PSOMxg3ZvtXD6vNuhyREQCE5orVAFKCmJMn1Cig6oikvNCFe4A8yaP0+mQIpLzQhfu8yeXs/PgUQ529gRdiohIYEIY7rpSVUQktOGurhkRyWWhC/cJpQVMKi/UQVURyWmhC3dI7L2v1Z67iOSwUIb7vMnlNLce4WhPf9CliIgEIpThPn/yOOIO63dp711EclMow33hlMRtCF5rORhwJSIiwQhluE8qL6J2XAGv7lC4i0huCmW4A5xTX8ErCncRyVGhDfez6yvY1tbJgQ5dqSoiuSe04X5OfQUAr6rfXURyUGjDfeGUCsxQ14yI5KTQhntpQYxZNaUKdxHJSaENd0h0zby646CeqSoiOSfU4X52fQUHOnvZvr8z6FJEREZVqMP92EFVdc2ISK4JdbjPqS2jMC+icBeRnBPqcI9FI5xVV65wF5GcE+pwh0TXzLo32+npiwddiojIqMmBcK+kpy/Oxt26Q6SI5I7wh3tD4qDqy28cCLgSEZHRE/pwr6soYnJ5IY3bFO4ikjtCH+4A500fz0vb9utiJhHJGbkR7tPG03q4mzfadDGTiOSGnAl3gFXb9gdciYjI6MiJcJ9VU0p5UZ7CXURyRk6EeyRinDetklU6qCoiOSInwh1gybTxbN3XQevh7qBLEREZcWmFu5ldaWabzKzJzO44QZuPmtl6M1tnZj/NbJnDd6zfvVFdMyKSA4YMdzOLAncDVwHzgBvNbN6ANrOALwMXuvt84AsjUOuwnFVXTmFehJcU7iKSA9LZcz8faHL3ZnfvAR4ErhvQ5jPA3e5+AMDd92a2zOHLj0U4p75CFzOJSE5IJ9zrgB0p4y3JaalmA7PN7Dkze9HMrhzsg8zsVjNrNLPG1tbW06t4GM6bNp51bx7iSHffqC9bRGQ0ZeqAagyYBVwM3Ah838wqBjZy92XuvsTdl1RXV2do0ek7b9p44g5/0n1mRCTk0gn3nUB9yviU5LRULcByd+91963A6yTCfkxZPLWSaMR4sbkt6FJEREZUOuG+CphlZtPNLB+4AVg+oM2vSey1Y2ZVJLppmjNYZ0aUFsQ4p76C57Yo3EUk3IYMd3fvA24DHgM2AA+5+zozu9PMrk02ewxoM7P1wFPAl9x9TCbohTMnsKblIIeO9gZdiojIiEmrz93dV7j7bHef6e5fS077qrsvTw67u3/R3ee5+1nu/uBIFj0cF55RRdxR14yIhFrOXKF6zKKGSoryojzftC/oUkRERkzOhXt+LMJ508er311EQi3nwh3gojMm0LT3CLsPdQVdiojIiMjJcH/3zCoAnt+irhkRCaecDPd5k8ZRWZzHc03qmhGRcMrJcI9EjHfPrOK5pn16rqqIhFJOhjvAu8+YwO72Lpr3dQRdiohIxuVsuF90RqLf/TmdEikiIZSz4d4wvpiG8cU8vWn0704pIjLScjbczYxL59bw/JZ9dPX2B12OiEhG5Wy4A1wyt4au3jgv6IImEQmZnA73pdPHU5QX5cmNY+7BUSIiw5LT4V6YF+XCM6p4cuNenRIpIqGS0+EOcOncGnYePMrmvUeCLkVEJGNyPtwvmZt43J+6ZkQkTHI+3CeVF3HmpHEKdxEJlZwPd4BL51bz8hsHONSppzOJSDgo3En0u/fHnWc264ImEQkHhTtwTn0l40vy+cOGPUGXIiKSEQp3IBoxrjizlic37KW7T1erikj2U7gnXXnWRA539+lGYiISCgr3pAtnVlFWGOORNbuDLkVEZNgU7kn5sQiXn1nL4+v30NsfD7ocEZFhUbinuGrBRA4d7eXFZt1ITESym8I9xZ/NrqY4P8oja9U1IyLZTeGeojAvyiVza3h83W7647qRmIhkL4X7AFctmMi+Iz2s2rY/6FJERE6bwn2AS+bUUBCL8MiaXUGXIiJy2hTuA5QUxLjszBp++9ounTUjIllL4T6I68+po62jh2c364ImEclOaYW7mV1pZpvMrMnM7jhJuw+ZmZvZksyVOPounlNDRXEev1q9M+hSREROy5DhbmZR4G7gKmAecKOZzRukXRnweWBlposcbfmxCNcsnMTj63dzpLsv6HJERE5ZOnvu5wNN7t7s7j3Ag8B1g7T7e+AuoCuD9QXmg4um0NUb51Gd8y4iWSidcK8DdqSMtySnHWdmi4F6d/9dBmsL1OKGCqZOKOZXq1uCLkVE5JQN+4CqmUWAbwC3p9H2VjNrNLPG1tax/WAMM+P6c+p4fksbuw+F4o8REckh6YT7TqA+ZXxKctoxZcAC4Gkz2wZcACwf7KCquy9z9yXuvqS6uvr0qx4l1y+qwx1+84oOrIpIdkkn3FcBs8xsupnlAzcAy4/NdPdD7l7l7tPcfRrwInCtuzeOSMWjaHpVCYsaKvj5yy2463YEIpI9hgx3d+8DbgMeAzYAD7n7OjO708yuHekCg3bjeQ007T3Cqm0Hgi5FRCRtafW5u/sKd5/t7jPd/WvJaV919+WDtL04DHvtx1xz9iTKCmI88NL2oEsREUmbrlAdQnF+jA8uruN3a3ZxoKMn6HJERNKicE/DTUsb6OmL84s/6bRIEckOCvc0zJ04jkUNFfz0pe06sCoiWUHhnqabzm+gubWDl7bqPu8iMvYp3NN0zcLJlBXG+MlKHVgVkbFP4Z6movwoHzm3nhVrdumKVREZ8xTup+BTF04j7s4PX9gWdCkiIielcD8F9eOLuXLBRH7y4ht06FbAIjKGKdxP0S0XzaC9q0+nRYrImKZwP0XnTq1kUUMF9z27lf64TosUkbFJ4X4aPn3RDLa1dfLEhj1BlyIiMiiF+2l43/xaplQW8f0/NgddiojIoBTupyEWjfCZ98xg1bYDvLClLehyRETeQeF+mj52Xj01ZQV864nXgy5FROQdFO6nqTAvymffO5MXm/ezsll77yIytijch+GmpQ1UlRbwrSc2B12KiMjbKNyHIbH3PoPnt7SxaptuKCYiY4fCfZg+vnQqVaX5fPMP6nsXkbFD4T5MRfmJvvfnmtp4dvO+oMsREQEU7hlx8wVTmVJZxD+s2EBcV62KyBigcM+AwrwoX3rfHNbvaudXq3cGXY6IiMI9Uz6wcDILp5TzT49voqu3P+hyRCTHKdwzJBIx/tvVZ7LrUBf3Prs16HJEJMcp3DPoghkTuPzMGu55egt72/W0JhEJjsI9w77y/nn09MX52ooNQZciIjlM4Z5h06tK+OzFM/nNK2/yXJNOjRSRYCjcR8DnLp7J1AnF/I/frKW7TwdXRWT0KdxHQGFelL+7dj7NrR18/xnd811ERp/CfYRcPKeGq8+ayL8+2URz65GgyxGRHKNwH0F/84H5FOZFuf3nr+p5qyIyqhTuI6h2XCF3Xjef1dsPskzdMyIyitIKdzO70sw2mVmTmd0xyPwvmtl6M3vNzJ4ws6mZLzU7XXv2ZK5aMJF/+f3rbNzdHnQ5IpIjhgx3M4sCdwNXAfOAG81s3oBmq4El7r4QeBj435kuNFuZGf/z+gWMK4rxxZ+9Sk9fPOiSRCQHpLPnfj7Q5O7N7t4DPAhcl9rA3Z9y987k6IvAlMyWmd0mlBbwDx88i/W72vnHRzYGXY6I5IB0wr0O2JEy3pKcdiK3AI8MNsPMbjWzRjNrbG1tTb/KEPh38yfyyXdP477ntvLo2t1BlyMiIZfRA6pmdjOwBPj6YPPdfZm7L3H3JdXV1ZlcdFb48tVzOXtKOV96+FW2t3UO/QMiIqcpnXDfCdSnjE9JTnsbM7sc+Apwrbt3Z6a8cCmIRfnOTYsx4HM/fVm3BhaREZNOuK8CZpnZdDPLB24Alqc2MLNFwPdIBPvezJcZHvXji/nnj57D2p3tfPmXa3DX+e8iknlDhru79wG3AY8BG4CH3H2dmd1pZtcmm30dKAV+bmavmNnyE3ycAFfMq+X2K2bzq9U7+e7TW4IuR0RCKJZOI3dfAawYMO2rKcOXZ7iu0Lvt0jNoaj3C1x/bxIyqEq46a1LQJYlIiOgK1YCYGXd9aCGLGir464deYfX2A0GXJCIhonAPUGFelGWfWEJNWSGfun8Vm3YfDrokEQkJhXvAqssK+L+3LCU/GuET965kx36dIikiw6dwHwMaJhTz41uW0t0X5+M/WMkePX9VRIZJ4T5GzJlYxv2fOo+2I9187HsvsPPg0aBLEpEspnAfQxY1VPLjTy+lraOHj/7bC7zR1hF0SSKSpRTuY8zihkoe+MwFdPT08dHvvUDTXh1kFZFTp3AfgxbUlfPgrRfQH4c//+7zvNjcFnRJIpJlFO5j1NyJ4/jV595NdVkBn7h3Jb9e/Y7b+YiInJDCfQyrH1/ML//jhSxuqOQLP3uFbzy+Sc9iFZG0KNzHuPLiPH50y/l8+NwpfPvJJv79/as42NkTdFkiMsYp3LNAQSzK1z+8kK99cAHPb9nHNf/6LGtaDgVdloiMYQr3LGFmfHzpVB76D++iP+78+T3Pcc/TW9RNIyKDUrhnmUUNlTzy+fdwxbxa7np0Izcue1G3LBCRd1C4Z6GK4nzuvmkx//yRs1m/q533ffMZ7n12q/biReQ4hXuWMjM+dO4UHv3Ce1g6fTx//9v1XH/3c6zdqb54EVG4Z70plcXc98nz+M5Ni9h1qIsPfOdZ/uvDr7H3sG4+JpLLFO4hYGZcs3AyT9z+Xj590XR+ubqFS77+NHc/1URHd1/Q5YlIACyoBzQvWbLEGxsbA1l22G3d18H/WrGBx9fvYUJJPp9970xuvmAqRfnRoEsTkWEys5fdfcmQ7RTu4bV6+wG+8fvX+ePmfVSV5vOpC6dz89KplBfnBV2aiJwmhbsct7K5jbuf3sIzr7dSkh/lhvMb+MQFU5lWVRJ0aSJyihTu8g7r32xn2TNb+O1ru+iLO382u5qblzZwydwa8qI6/CKSDRTuckJ727t4cNUOfrpyO7vbu6gqzef6c+r40LlTmDuxDDMLukQROQGFuwyprz/OU5ta+cXLLTyxcQ+9/c4ZNaW8/6xJvH/hJGbVlCroRcYYhbuckv0dPfzutTf53ZpdrNy6H3eYNqGYy86s5bIza1gydTz5MXXdiARN4S6nbe/hLh5bt4cnNuzh+aY2evrjFOdHedeMCVw0q4p3zZzA7JoyIhHt1YuMNoW7ZERHdx/PNu3j2c37+OPmVra1JW5SVlmcx3nTxnPu1EoWT63krLpyCvN0Hr3ISEs33GOjUYxkr5KCGO+bP5H3zZ8IwI79nazcup+VzW28tG0/j6/fA0AsYsyuLeOsunIWTCln/uRxzKkto6RAv2IiQdCeuwzLviPdrN5+kNXbD7Bm5yHW7DzEwc7e4/Mbxhczu7aMM2pKOaOmlJnVJUyvKqGiOD/AqkWyl/bcZVRUlRZwxbxarphXC4C703LgKBt3H2bjrnY27G5n854j/L/X99Lb/9aOREVxHlMnlFBfWUT9+GLqK4uZXFFIXUURkyqKKNUev8iwpPU/yMyuBL4FRIEfuPs/DphfAPwIOBdoAz7m7tsyW6pkAzNLhPX44uOBD9DbH2f7/k6aWzvYtq+DrW0d7NjfyZqdh3h07W76BtyLvrQgRu24AiaWF1JTVkh1WQHVpQVMKM1nQmkBE0ryqSzJZ3xxvu6ZIzKIIcPdzKLA3cAVQAuwysyWu/v6lGa3AAfc/QwzuwG4C/jYSBQs2SkvGmFmdSkzq0vfMa8/7uxu72LXwaPsPHiUNw92sac98drd3sWqbftpPdxNd1980M8uiEWoLM6nojiPcUV5lBflMa4wj7LCGOMKY5QWxigrzKOkIEZpQZSS/BglBTGK8hPDRflRivKi5EVN5/VLaKSz534+0OTuzQBm9iBwHZAa7tcBf5scfhj4jpmZB9WhL1klGjHqKoqoqyjiRB2J7s7h7j7ajvTQdqSbfUd6ONjZw/7OHg529nKws4dDR3s50NlLy4GjtB9tp/1oL0d6+kj3tzAWMQrzoslX5Ph7QSxKQSySfEXJj0XeekUT73lRIy8aSb4Sw7FohLyIJd6jRjRixCJGNBIhFk0OW3J61Igkh4+9v23YDDOOT4tESLybEbHEX0wRe2uaHR9+a56+uHJLOuFeB+xIGW8Blp6ojbv3mdkhYAKwLxNFipgZ4woTe+TTT+GGZ/G409nbz+GuXjq6+zjS3U9Hdx+dPf109vTR0Z147+rtp7Onn67eOEd7++nq7ae7LzF+7P1wVx/dff1098Xp7YvT0x9PDPfH6e33rHjMoRkYKYFPYoIdn/fWF4NBYgZv/Uzqz781/finHx9OnZ78JFK/W1I/Y2B9gw5jJ5ie2n7wL68TfqWd5Lsuna/B4XxZfv6yWXzg7Mmn/fPpGNWjVmZ2K3ArQENDw2guWnJUJGKUFsRG5QBtPO70xhNB39sXpy/u9MXj9PU7vf1x+uN+/EugL56Y3x8/Nu7Ej417ctgT43F34nHod8fd6U8Zjscdh+NfLPHkfMdxT0x3T4zHHfDE+7H5x4aT/3A/1j7xWSR+JPnuyTbJ8eRnJIZJ+QspZXpK2+Nzk5/P237m7W1OMEhqZ8Dbpw++TU70dXuyToW0vqKH+T1eXjTyt91O5zd+J1CfMj4lOW2wNi1mFgPKSRxYfRt3XwYsg8SpkKdTsMhYFYkYBZEoBTGgIOhqJNelc7OQVcAsM5tuZvnADcDyAW2WA3+ZHP4w8KT620VEgjPknnuyD/024DESp0Le5+7rzOxOoNHdlwP3Aj82syZgP4kvABERCUhaHZHuvgJYMWDaV1OGu4CPZLY0ERE5XbqHq4hICCncRURCSOEuIhJCCncRkRBSuIuIhFBg93M3s1bgjdP88Spy89YGubjeubjOkJvrnYvrDKe+3lPdvXqoRoGF+3CYWWM6N6sPm1xc71xcZ8jN9c7FdYaRW291y4iIhJDCXUQkhLI13JcFXUBAcnG9c3GdITfXOxfXGUZovbOyz11ERE4uW/fcRUTkJLIu3M3sSjPbZGZNZnZH0PWMBDOrN7OnzGy9ma0zs88np483s9+b2ebke2XQtWaamUXNbLWZ/TY5Pt3MVia398+St50OFTOrMLOHzWyjmW0ws3flyLb+6+Tv91oze8DMCsO2vc3sPjPba2ZrU6YNum0t4dvJdX/NzBYPZ9lZFe4pD+u+CpgH3Ghm84KtakT0Abe7+zzgAuCvkut5B/CEu88CnkiOh83ngQ0p43cB/+LuZwAHSDyMPWy+BTzq7nOBs0msf6i3tZnVAf8ZWOLuC0jcTvwGwre97weuHDDtRNv2KmBW8nUrcM9wFpxV4U7Kw7rdvQc49rDuUHH3Xe7+p+TwYRL/2etIrOsPk81+CFwfTIUjw8ymAO8HfpAcN+BSEg9dh3CucznwZySeiYC797j7QUK+rZNiQFHy6W3FwC5Ctr3d/RkSz7hIdaJtex3wI094Eagws0mnu+xsC/fBHtZdF1Ato8LMpgGLgJVArbvvSs7aDdQGVNZI+SbwX4B4cnwCcNDd+5LjYdze04FW4P8ku6N+YGYlhHxbu/tO4J+A7SRC/RDwMuHf3nDibZvRfMu2cM8pZlYK/AL4gru3p85LPsYwNKc6mdk1wF53fznoWkZZDFgM3OPui4AOBnTBhG1bAyT7ma8j8eU2GSjhnd0XoTeS2zbbwj2dh3WHgpnlkQj2n7j7L5OT9xz7My35vjeo+kbAhcC1ZraNRHfbpST6oiuSf7ZDOLd3C9Di7iuT4w+TCPswb2uAy4Gt7t7q7r3AL0n8DoR9e8OJt21G8y3bwj2dh3VnvWRf873ABnf/Rsqs1AeR/yXwm9GubaS4+5fdfYq7TyOxXZ90948DT5F46DqEbJ0B3H03sMPM5iQnXQasJ8TbOmk7cIGZFSd/34+td6i3d9KJtu1y4C+SZ81cABxK6b45de6eVS/gauB1YAvwlaDrGaF1vIjEn2qvAa8kX1eT6IN+AtgM/AEYH3StI7T+FwO/TQ7PAF4CmoCfAwVB1zcC63sO0Jjc3r8GKnNhWwN/B2wE1gI/BgrCtr2BB0gcU+gl8VfaLSfatoCROBtwC7CGxJlEp71sXaEqIhJC2dYtIyIiaVC4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJC/x+rCZK6/8hWxwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9, last_epoch=-1)\n",
    "\n",
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "    scheduler.step()\n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.循环模式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "###　余弦退火"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x118f1cf90>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4G/d5J/DvC4DgTQAEwRPgKZE6eYiULNqJpdhuYktZK90ctRvbaTe7Tjd1m26yzSbbXbdN22c3abZN+9TbxNukqd06tpvmUGM5TmzHcmpLskhRJHVSIkUSPCSeAG/i+u0fwNAMQ4ogOcBc7+d5/JgAh4MfNL95Z+b9XSSEAGOMMX0xKV0Axhhj8uPgzhhjOsTBnTHGdIiDO2OM6RAHd8YY0yEO7owxpkMc3BljTIc4uDPGmA5xcGeMMR2yKPXBeXl5ory8XKmPZ4wxTWptbR0TQrjW206x4F5eXo6WlhalPp4xxjSJiPri2Y7TMowxpkMc3BljTIc4uDPGmA5xcGeMMR3i4M4YYzq0bnAnom8R0QgRXVjj90REf01E14mog4j2yV9MxhhjGxHPnfu3Adx/m98/AGB77L/HAfzt1ovFGGNsK9YN7kKINwFM3GaTYwCeEVGnAdiJqEiuAq7UMeDD37x+Dd6JuUR9BFMxIQRa+ybwzX+7ga5b00oXhynk8vAU/u7nPTjv9YGXCl2dHIOYSgB4l70eiL03vHJDInoc0bt7lJaWburD3u4ex1d/0oWv/qQL+8sd+GijBx9tcoOINrU/pg3++SC++fMe/OD8EPqXXdh3FeXg3+8rwSfuLEeKmZuQ9GwhGMY/vN2L750bxNVlF/aKvEx8qL4En3xvBbJSFRuXqTpJ/ZcQQjwN4GkAaGpq2tTl9rcOVeGDtUX44fkh/KBtEJ//lw50j83giw/slLWsTD1mFkN47FvvoHPAh7u25eEz927H/vJcvH7lFr5/fgh/+tJltA/48bVfq4fZxBd5PQqGI3jiuTa8evkWGssc+JMP7cHhahdOdY/j+22D+NprXXirewzP/IcDSEsxK11cVZAjuA8C8Cx77Y69lzBuRwZ++33b8OnDVXjyhxfxjZM9sKWn4NOHtyXyY5kCFoJhPP5MCy4M+vH1Rxrx/t2FS7/7jbsq8Bt3VeDrJ7vxv1++guw0C/7sQ3v4KU5nIhGBz3+3A69evoUvHduNx5rLl37nyc3Ax/Z78MPzg/i9F87j0/90Dt94tJGf4iBPV8jjAB6L9Zo5CMAvhPillEwiEBH++MHd+FB9Mb7y46t49nRcUy4wjQiFI/id77RFU3Efrf2FwL7cbx2qwn8+XIXnzvTjyz++muRSskQSQuCP/vUivt82iP/6/upfCOzLHasvwZ9+aA9evzKCz73YjnCE8/Dr3rkT0XcAHAaQR0QDAP4QQAoACCG+DuAEgCMArgOYA/CbiSrsakwmwp9/tA4ziyE8+cML2FmYjaby3GQWgSXI1169hp9euoU/fnA3frXBfdttP/+BGkzNB/H1k93YWZSNY/UlSSolS6QXznrxzKk+PH53JX77fbd/Mv/4HWWYmg/hyz++gh1F2YZ/kielWpqbmpqEnLNCzgVCuOerJ5Gfk4offPoumDj3qmneiTnc+xcncWRPIb72UENcfxOOCBx76t8wPhPA6587jHQr5161bGohiPf9+RuodGXixU81x51ue/yZFrx1fQw/+6+HkZ+TluBSJh8RtQohmtbbTjeJqQyrBZ+/vwYdA358vy2hKX+WBP/7x1dgIuC/PbAj7r8xmwj/8+guDPsX8PSbPQksHUuGp352HRNzATz5wd0bakf570d2IhCO4Ks/MXaKTjfBHQA+VF+COrcNX3nlCuYCIaWLwzbpbO8EXuoYxm8dqkKRLX1Df3tHpRNH9hbi6ye7cdO/kKASskTrH5/D3/9bLz68z429btuG/rY8LxO/eVcF/rl1ABcG/QkqofrpKribTIQn/90u3JpaxNdP8p2bFkUiAn/yo0sosqXhU3dXbWofX3xgJ8IRga+8ckXm0rFk+V8vX4bFTPj9D9Rs6u+fuGcbcjOs+JMfXTLsICddBXcAaCzLxb+rK8Y3TnZj2D+vdHHYBv3g/CA6Bvz4b/fv2HTO3JObgU++twLfOzeIjgGfzCVkifbOjQm8fOEmPn24CgWbzJnnpKXgs++vxpkbE3jl4i2ZS6gNugvuQLTnRDAcwbOnuGuklggh8PSbPdhRmI0H64q3tK9PH65CdpoFf/fzGzKVjiXL0292w5lpxX98b+WW9vNrTR6UOzPw9JvdMpVMW3QZ3D25GbhnRwFeOOvFYiisdHFYnFr6JnHl5jQ+cWf5lns7Zael4MP73Hj5wjBGpxdlKiFLNO/EHF67MoKHDni2PNLUYjbhkYNlONfvM2TuXZfBHQAeay7D+GwAL3feVLooLE7PnupDdpoFx+q3dtcuebS5DMGwwAtn+2XZH0u8597pBwH49TvKZNnfRxs9SEsx4R8NOMBRt8H9PdvyUJGXyaNWNWJ0ehEvXxjGRxrdyLDKM+VRlSsL79mWh+fO9CMUjsiyT5Y4i6EwXjjrxX07C1Bi31gvqbXYMlLwofoS/OD8IPzzQVn2qRW6De4mE+Hjd5SitW8SF4eM90imNS+c7UcwLPDIQXnu2CSPHCzDkH8Br10ZkXW/TH4nOocxMRvAo83y14GFYATfbR2Qdb9qp9vgDhj7kUxLQuEInjvTj/dsy0OVK0vWfd+3Mx9FtjSuAxrw7Kk+VOZl4q6qPFn3u6fEhn2ldvzj6T5EDDTnjK6Duy0jBcfqSvCDtiHDPZJpyWtXRjDkX5D9jg2INqr9+oFS/PzaGHpGZ2TfP5PHhUE/zvX78PGDZQmZOuTR5jLcGJvFW91jsu9brXQd3IHoQZ0PhnG8fUjporA1vHjWi8KcNNy7Iz8h+3/oQCksJsILLd71N2aKeLHFi1SLCR9pvP0EcZt1ZG8RcjOteP6sceqA7oP77uIcVLoycaIjKbMQsw3yzwfx5rVRfLC2CJYEzcHtyk7FndvycKJz2LCjFdUsHBE40XkT9+zIhy09JSGfkWox4/49hXj98gjmA8boHq374E5E+ODeIpy5Mc79nVXo1Uu3EAwLHK1N2LK7AIAP7i2Cd2IeFwanEvo5bOPO9k5gbGYxKXVgPhjGG1eN0biu++AOAEdqixARwI8vcp93tXmpcxgl9nTUe+wJ/Zz37y6AxUT4USen59TmROcw0lJMuCdBaTnJgYpcODOteKnTGE/xhgjuNQXZqOLUjOr454P4+bVRHNlbmPCl8ewZVk7NqJCUknlfTb5s4xvWYjGb8IE9hXjNIKkZQwR3IsJRTs2ozk9jKZkjexP7OC6RUjOdBhyKrlbJSslIjJSaMURwB4CjtcWcmlGZE0lKyUik1IxRHsu14KWO5KRkJFJq5kcGqAOGCe7VBVmocmXipQ7OuapBMlMyEnuGFXdty8NLHZyaUYNwRODlC9FeMolOyUgsZpNhes0YJrgTEY7WFuOdGxMYmeYVepT206VeMvJMEhavo3uLMDDJqRk1kFIyyUrLSY4aJDVjmOAORA9qRAA/Mejk/WryciwlU7fBJdS2SkrNnODZQhX3cpJ6yax0oCIXeVlWnLig7zpgqOBeXZAFT2463rg6qnRRDG0hGMbb3eO4b2d+0lIyEnuGFU3lDt3ftWnBG12juKsqL2kpGYnFbMLhmny82TWKsI7nmjFUcCciHKp24e3uMV7EQ0FneycwHwzjUI1Lkc8/VJ2PKzeneQFtBd0Ym0Xf+JyCdcAF/3wQ5736XYbRUMEdAA5X52MuEEZL76TSRTGsN66OwmoxoblS3tn/4nU4FlBOdvHdu1KkJ6fD1clNyUjeuz0PJgJO6vgJznDBvbnKCavZhJNdnJpRysmuUdxRkbvpBbC3akdhNgpz0rgOKOhk1ygq8zJR6sxQ5PPtGVY0lDp0XQcMF9wzUy3YX8E5V6UMTM7h+sgMDlUr8zgOvJue+/m1MV6hSQELwTBOdY/jbgXrABBNzXQM+jE+o8+BjYYL7kD0UbDr1gyGfPNKF8VwpMbswzXKPI5LDtW4ML0Qwrl+/eZc1ep0zzgWQ5Gl9JhSDte4IATw5jV93r0bMrgfWsq56vOgqtnJrlGU2NNR5cpUtBx3bcuD2UScd1fAya5RpFpMOFjpVLQce4ptcGZacVKnvecMGdy352eh2JbGqZkkC4QiePv6GA7XuJLeBXIlW3oKGksd3C1WASevjuJgpRNpKcq0uUhMJsLd1S68eW1Ml8vvGTK4ExEO1eTjrevjCIQ455osLb0TmA2EFU/JSA7VuHBxaIpHLCdR//gcesZmFU/JSA7XuDAxG0CHDkcsxxXcieh+IrpKRNeJ6Aur/L6UiH5GRG1E1EFER+QvqrwOVbswsxjCuX7uEpksJ7tGkWImNFcp+zgukRp13+wyzrqaSpPSYEo2qC/33u0uEEGXqZl1gzsRmQE8BeABALsAPExEu1Zs9j8AvCiEaADwEID/K3dB5XbXNicsJsKbnHdPmpNdo2gqy0VWanJHJK5ld3EOXNmpXAeS6GTXGDy56ajIU7bNRZKbaUWt267LRtV47twPALguhOgRQgQAPA/g2IptBICc2M82AKqfejE7LQW1bhtO9YwrXRRDmJgN4MrNady1TR137UA0Pddc6cSpnnGeJTIJwhGBMzfGcVdVnuJtLsvdWeVEu9eH2cWQ0kWRVTzBvQTA8iXDB2LvLfdHAB4hogEAJwD8jiylS7CDlU50DPh1d1DV6J0b0Yuo0j0kVjpY6cTo9CJ6xmaVLoruXR6ewvRCSJV1IBQRaO3TV4pWrgbVhwF8WwjhBnAEwLNE9Ev7JqLHiaiFiFpGR5V/DGquciIcETjbO6F0UXTvVPc40lPMqHUnZ2GOeEn5/1Pd/ASXaNK/sdqCe1OZAxYT6e4pPp7gPgjAs+y1O/becp8E8CIACCFOAUgD8EsThwghnhZCNAkhmlwu5RtUGsscSDETTvdwcE+00z0TaCp3wGpRVwetcmcGCnJScVpnJ7Yane4ZR0VeJgptaUoX5RdkplpQ67bprg7Ec6adBbCdiCqIyIpog+nxFdv0A7gXAIhoJ6LBXflb83VkWC2oc9t1d1DVZnxmEVdvTavujg14N+9+umeC8+4JFI4IvHNjQpV1AIg+wektRbtucBdChAA8AeAVAJcR7RVzkYi+REQPxjb7HID/RETtAL4D4DeERs6Ug5VOdA76MaOjg6o2Z25En4zUemIfrHRibGYR3aMzShdFty4O+TG9GMLBylyli7Kqg5X6S9HG9YwshDghhKgWQlQJIf4s9t6TQojjsZ8vCSHuEkLUCSHqhRA/SWSh5cR598Q73TOODKsZtUledSle0kXnFKfnEkZ6Om5W6QW+qSxXdyladSVAFbCv1AGr2YTT3KCWMKe6x7G/PBcpZnVWtzJnBopsaVwHEuhU9ziqXJnIz1FXvl2SbjWj3mPXVaOqOs+2JJIOKufdE2NsZhHXRmZUm5IBonn3g5VOnOb+7gkRCkdwtndS1XUAiD7BXRj0Y3ohqHRRZGH44A4AB6uiefcpnRxUNVl6HFfJlANraa50Ynw2gGsjnHeX24WhKcwshjRRB8IRoZtV2ji4AzhYmYuIiE5sxeR1umccWakW7CnOWX9jBUl3lfwEJz/p3/SOCnUH931lsRStTuoAB3fE8u4Wk64aU9RC6t9uUWm+XeLJTUeJPV03J7aanO4Zx7b8LLiyU5Uuym2lpZhRX6qfFK26z7gkSUsxo7bExj1mZDY5G8D1kRnsL1dn97fliAhN5Q6c7Z3kvLuMIrFh/VqoAwCwv9yBC0NTmAtov2s0B/eYxnIHLgz6sRAMK10U3ZDm6mgqcyhckvg0ledidHoR3gleflEuXSPTmF4IaaoOhCMC573aX36Rg3tMU1kugmGBjgH9TdqvlJa+SaSYCXUedc0nsxYpALX08ROcXKTGyaZybQT3faUOEAGtOmhU5eAe08gntuxa+yawp8Sm+HJq8aouyEZ2qgUtOpsdUEmtfZNwZaeiNDdD6aLExZaegur8bF3UAQ7uMbmZVlS5MnXTDUppi6Ew2gf8mnkcBwCzidBQ5tDFXZtanO2dQFOZQ1Xzt6+nsdyBc32TCGt8XVUO7ss0leWitW9Sl4vlJtuFQT8CoQgay7TRkCZpKnOga2Qa/nke87BVt6YWMDA5v/RUrBVNZQ5ML4bQdWta6aJsCQf3ZRrLHfDPB3kCKRlIT0BaPLGFAK+tK4N38+1au8BHy6v11AwH92XebVDT9kFVg5a+SZQ7M1Tft3ml+lI7zCbi1IwMWvomkJZiwm6VD2BbyZObjvzsVLRqvGs0B/dlKvIy4cy0ct59i4QQONc3qbmUDBCd4393cQ43rMugtW8SdW67aieMW4s05kHrN3na+ldPMCLCvjIHWvnE3pIbY7MYnw1opvvbSo1lDpz3+hAMR5QuimbNBUK4ODSl4TqQi4HJedyaWlC6KJvGwX2FpjIHesfnMDq9qHRRNKtFY4OXVmoqy8VCMIKLQ1NKF0Wzznt9CEfEUv5aa5ZStBp+iufgvoJ0p8F375vX0jsBW3oKqlxZShdlU6Q6wBPJbZ4UFPeVavMCv6s4B+kpZk1PScLBfYU9JTZYLSZNX7GV1tI3icYyB0wm7fRtXq4gJw1uR/rS9Als41r6JlFdkAVbRorSRdmUFLMJdR6bpusAB/cVUi3RScTadDC3hBJ8cwH0jM5qrgvkSk1lDrT1cx3YjEhE4Hz/pA7qQC4uD09hPqDN+aY4uK+i3mNHZ2wQDtsYacKlBo3MJ7OWeo8dN6cWMOznScQ2qmdsFlMLITR4tB3c6z12hCICF4a0Od8UB/dVNJQ6EAhFcHmYG9Q2qq3fByKgVuPBvSGWK+a7941riw0AayjVdh2oj5W/TaMD2ji4r0KqlHqY9jPZznt9qCnIRlaqRemibMnOohxYLSbNnthKavP6kJ1q0WyDuiQvKzrhmVYv8BzcV1FkS0NBTiqf2BsUic2DrfU7NgCwWkzYU5zDF/hNON/vQ32pXbMN6svVe+yarQMc3FdBRGjwOLhRdYNujM/CPx/UfK5V0lDqQMeAnwczbcBcIIQrN6c03+YiaSi1Y9ivzbYXDu5rqC+1o298DuMzPJgpXtLja70O7tyB6Im9GIrgyrC2ZwdMpo4BPyJCT3UgeqNyXoOpGQ7ua5DuPNoHtHdQlXLeO4nsVAu2aTzXKllqVPVyei5eSxd4nTy97ZLaXjT4FM/BfQ173TaYTaTZxhQltPX7UOfRR64VAIptaXBlp3Id2IC2/uhsoLmZVqWLIgurJTqrpRbb3zi4ryHDasGOwmw+seM0Hwjjys1pXTSmSqJtL9ptUEs2IQTavL6lJx69aPA40DmovbYXDu630VBqR7vXxyszxaFz0I9wROgquAPR1MyNsVlMzgaULorqDfkXMDq9qMM6YMdCMIKrN7XV9sLB/TbqPdHltnhlpvVJj611bv2d2ACPeYiHVAfqddJTRtKg0cFMHNxv492Dyif2etr6fShzZsCZpa2Vl9azt8QGE2nvxFZCW78PqRYTdhRqa+Wl9ZTY05GXpb22l7iCOxHdT0RXieg6EX1hjW0+RkSXiOgiET0nbzGVUeHMhC09hXtLxKHNO6mbvs3LZaZaUFOYo8neEsnW1j+JvbFZVfWEiNBQatdcHVj3KBCRGcBTAB4AsAvAw0S0a8U22wF8EcBdQojdAH4vAWVNOpOJUOex47xXmxMHJctN/wJuTS2iTofBHYimGbjt5faC4QguDE3pug7cGJuFb047bS/xXGIPALguhOgRQgQAPA/g2Ipt/hOAp4QQkwAghBiRt5jKqXfb0HVrGnOBkNJFUS0pH623XKuk3mPD1EIIveOzShdFta7enEYgFNFxHYh+r44B7dzoxRPcSwB4l70eiL23XDWAaiJ6i4hOE9H9q+2IiB4nohYiahkdHd1ciZOszmNHOCJ4ybXbaB/wIcVM2Fmkr1yrpI4HtK1L7xf4vW4bAKBdQ6kZuZJjFgDbARwG8DCA/0dEv3SUhRBPCyGahBBNLpdLpo9OrNpY7w8tHdRka/f6sLMoB2kpZqWLkhDb87ORYTWjndNza2r3+pCbaYXbka50URIiJy0FVa5MTV3g4wnugwA8y167Y+8tNwDguBAiKIS4AaAL0WCvea7sVJTY07kr3BoiEYGOAb/uukAuZzYR9pTYuA7cRvuAD3VuG4j0MTp5NVL7mxDaaHuJJ7ifBbCdiCqIyArgIQDHV2zzA0Tv2kFEeYimaXpkLKei6j12TV2xk6lnbAYziyHdNqRJ6j12XBqa4tW5VjGzGMK1kRlD1IGxmUUM+ReULkpc1g3uQogQgCcAvALgMoAXhRAXiehLRPRgbLNXAIwT0SUAPwPw+0KI8UQVOtnqPDZ4J+Z5hshVSD2J6j02hUuSWHVuOwLhCK7c5LaXlToH/BACug/udRpL0caVcxdCnBBCVAshqoQQfxZ770khxPHYz0II8VkhxC4hxF4hxPOJLHSySQe1Y5Bzriu1e33ISrWgMk8fM0GupS528WrXUG+JZOmIPdXqOTUHADuKsmE1mzTzFK+v0QYJsic2SlErV+xkah/wRUdx6mQmyLWU2NPhzLRyHVhF+4APntx03cwEuZZUixk7i7I1Uwc4uMchM9WC7fnaOajJshAM4/KwfgeuLEcUHdDGdeCXtXv13aC+XJ3Hjs6B6CR5asfBPU51HhvaB7TTUp4Ml4enEAwL3efbJXVuO66PzmB6Iah0UVRjZHoBg7553fZvX6nObcdsIKyJyQQ5uMepzmPHxGwAA5PaW0sxUaS7WCPcuQPRC7wQ0emNWVRHrEHdOHVAO7OEcnCPk/TYqYWDmiztA37kZ6eiMCdN6aIkxbu9JTi4S9oHfDCbCLuL9Tk6eaXKvExkp1o0kZ7j4B6nmsJspFpMmjioydI+EF1WT88DV5ZzZFpR5szgOrDMea8P1QXZyLBalC5KUphMhFqPTRM9Zji4xynFbMKeEm0c1GTwzwfRMzqLOrcx8u2SOjcPaJMIIY1ONl4duDI8jYVgWOmi3BYH9w2oddtwYXAKIY2tpZgInQPGyrVKat02DPsXMDKtjVGKidQ3Pgf/fNCAdcCOUETg0rC6B7RxcN+AOrcd88EwrmugpTzRpLvX2hJjndhSIOvgvPu7dcBod+6x3mEdKk/PcXDfgFoNTvuZKO1eH8qdGbBlpChdlKTaXZwTHdDGqRm0e/1ItZhQXZCtdFGSqjAnDa7sVNWPVubgvgHlzkzkpFlUf1CToWPAb7jHcQDIsFpQXZDNdQDRaQf2lNiQYjZWGCEiTbS9GOuobJHJRKh125fm0jCqkakF3JxaWJrr3mjqYnXAyAPaQuEILgz5DZeSkdS5begZncWUige0cXDfoFq3TRMt5Ykk3bUarZeEpNZjg28uCO+EcQe0XRuZwUIwYphpB1aqjT21XlDxExwH9w2q82ijpTyR2r3SwBVjBvelAW0GfoIz2ujklaQbGzXXAQ7uG7Q0/a+BG1XbB6IDV9Kt+lxWbz3SgDZj1wE/ctIsKHdmKF0URdgzogPa1NxrioP7BhXa0pCfnaqpVdDlJIRA56DxBq4sl2I2YVdxjmHrABBtTK11G2d08mrU3v7GwX0TajXQUp4o/RNz8M0FDduYKqlz29E56DfkgLaFYBhXb04btjFVUue2YUjFA9o4uG9CvceGbpW3lCfK+aVcq8FPbI/NsAPaLg5NIRQRhs23S9Q+oI2D+yZId61qbilPlI4BYw5cWanWre4TO5GMsqzeeqQBbWpNzXBw34SlkaqGDO4+7C7OMdzAlZUqnLGpX1V6YidShzTVs80YUz2vRe0D2ox9hm6S1FJutGkIQuEIOgf9hs+3A9EBbXvdxpwltN3r4zoQUxurA2oc0MbBfZO0MPxYbksDVwyeb5fUebQx9auc/PNB9IwZb6rntdR57PDNBdE/Mad0UX4JB/dNqvPYDTf1q5RbrPc4FC6JOtRpZOpXOUlTPdeX8p07sGx1LhWmZji4b1L90rSf6juoiXLea+yBKyvVe4w3oM2oUz2vRc0D2ji4b9LuYhvMJjJUaqbda6xl9dZTaEtDQY76p36VU7vXh8q8TMNN9bwWNa/QxsF9k9JSzKgpyDbMgtnzgTCu3po2fPe3lercdkM1rLcP+Aw/eGmlWrdNlQPaOLhvQZ0nemKrsaVcbheH/AjzwJVfUuexo2dsFv45/Q9ou+lfwK2pRa4DK9R77FgIRtB1S10D2ji4b0G9x4aphRB6x9XXUi63pZGpfNf2C5by7oP6v3s/b/CZINfybqOquuoAB/ctkCq5ER7LOwb8KLalIT/H2ANXVtproKUX2wd8sJgIu4pylC6KqpQ5M2BLT1FdHeDgvgXb87ORYTUbIu/ePuDjO7ZV5KSloNKVifMG6DXV7vVhZ1EO0lKMOdXzWogIdR676uJAXMGdiO4noqtEdJ2IvnCb7T5MRIKImuQronqZTaTalnI5Tc4G0Dc+x8F9DfXu6Imt57aXSETE1s3ltNxq6t02dN2axlwgpHRRlqwb3InIDOApAA8A2AXgYSLatcp22QA+A+CM3IVUs3qPHReHphAIqaulXE7tPFHUbdV57BibWcSwX78D2nrGZjCzGOI6sIY6jx0RAVwYVM+Atnju3A8AuC6E6BFCBAA8D+DYKtv9CYAvA9BvDV9FrduGQCiCrlvTShclYdq9fhC9m19mv2hp6lcdP8G1x9JO/PS2uqVZQlVUB+IJ7iUAvMteD8TeW0JE+wB4hBAvyVg2TVhaT1Nl+TY5dQz4sM2VhaxUi9JFUaWdRdlIMZOu8+7tAz5kWs2ocmUpXRRVcmWnosSerqo4sOUGVSIyAfgLAJ+LY9vHiaiFiFpGR0e3+tGq4Hakw5lpVV1LuVyEENyYuo5Uixm7inJ0WweAaGPqXnd0VDZbXZ1HXe1v8QT3QQCeZa/dsfck2QD2AHiDiHoBHARwfLVGVSHE00KIJiFEk8vl2nypVUStLeVyGfTNY2wmwMF9HXWe6Hqa4Yj+GlUXQ2Hpntg8AAAXcUlEQVRcGp7iOrCOOrcd3ol5jM8sKl0UAPEF97MAthNRBRFZATwE4Lj0SyGEXwiRJ4QoF0KUAzgN4EEhREtCSqxCdW47ro/OYFqHy+5JF616bki7rTq3HbOBMLp1uOzepaEpBMOC68A6lsa9qOTufd3gLoQIAXgCwCsALgN4UQhxkYi+REQPJrqAWtBQaocQ0YE+etPW70OqxYQdRcZeVm89DbEpcNv6JxUuifza+qPBqqGUp3q+nVq3DSZ6999LaXHl3IUQJ4QQ1UKIKiHEn8Xee1IIcXyVbQ8b6a4dePeKrc8TexK1bpvhl9VbT0VeJmzpKao5seXU5vWhyJZm+GX11pNhtWBHYY5q6gCfsTKwpadgW36W7vLugVAEF4am+I4tDkSEhlJ9tr2c904uPZmw22sojU4mGFFB2wsHd5nUe+xo69fXKMXLw9HBWfXckBaXeo8dV29NY2ZRPaMUt2p0ehHeiXk08OpbcWkodWB6MaSKthcO7jJpKLVjfDYA78S80kWRjZRm4ru2+DSUOqJtLzq6e19qUOc6EJf6pRSt8nWAg7tMpDubNq9+8u5tXh8Kc9JQZEtXuiiaIPUmadNVcJ+ExUTYU8yjk+NRmZeJnDSLKuoAB3eZVBdkIcNqVsUVWy5t/T6+a98AW0YKqlyZumpYb+uPzgSZbuWZIONhMhHqSx2qqAMc3GViMZuwt8Smiiu2HMZnFtE/Mcf59g2q9zh0M0NkOCLQ7uUL/EY1eOzoUkHbCwd3GTWUOnBpyI+FYFjpomyZlGvlnjIb01Bqx9hMAAOT2m97uTYyjdlAmIP7BjWURmeIVHoSMQ7uMmootSMYFrg4pJ5pPzerrd8Hs4mwt4RzrRshBcJzKngs36rzsRRjPfeU2RDpaVfpbrEc3GXUoKPBTG3eSewozOZc6wbVFGQjPUUfbS9t/T7YM1JQ7sxQuiiaYs+wojIvU/E6wMFdRvk5aSixp2s+7x7Ntfr5cXwTLGYT9rr10fbS5p1Eg8cOIp4JcqPqS5Uf98LBXWb1pfalx1mt6h6NrrrDA1c2p6HUjstDU1gMabftZXohiGsjM9zmskkNpQ6MzSxi0Kdc2wsHd5k1eOwY9M3j1pR2F6Rq7ePBS1vR4HEgEI7gwqB2J5KL9vjhOrBZUopWOpeUwMFdZk3luQCAll7t5t1beifhzLSiIi9T6aJoUmNZ9G5X63XARNxbarN2FGYj02rm4K4nu4tzkJZiQkvfhNJF2bTWvgnsK3NwrnWTXNmpKHdmoEXBE3urWvsmsaMwh5dW3CSL2YSGUoeiF3gO7jJLMZtQ57YresXeitHpRfSOz6GpjO/YtqKxLBetfZOaHMwUCkdwrn8STeVcB7aiscyBKzenFFvEh4N7AjSVO3BxaAqzGpwdsDX2xMEn9tY0lTswMRtAz9is0kXZsCs3pzEXCC+ll9jmNJU7EBHKTSLGwT0Bmspzl4Zua01L7ySsFhP28OClLdkfuzi2ajDv3tIrXeBzFS6JtjWUOmAiKJae4+CeAPtKHSAFD+pWtPRNos5tQ6qFBy9tRWVeFuwZKZpse2npm0SRLTpmg21eVmp0ZaZWheoAB/cEsKWnoDo/W3PBfSEYxsUhPxrL+I5tq0wmQmOpQ3N1QAiBlt5JTsnIpKncgbZ+H0LhSNI/m4N7gjSWO9DWN4mwCpbbile714dgWHBjqkwayx3oGZ3FxGxA6aLEbdA3j5tTC1wHZNJY5sBcIIzLw9NJ/2wO7gnSVBZdbqvrVvIP6mZJd5l81yaPptgTkJZ6Tkll5Xy7PJbGvSiQmuHgniDSia2lx/LWvklUuTLhyLQqXRRdqHXbkGImTeXdW3onkWE1Y0dhttJF0YUSezqKbGmKxAEO7gniyU2HKzsVrb3aOLEjEYHWvsmlixLburQUM/aU2DTVY6albxL7Sh2wmDk0yKWpPBetvckf88BHMEGICPvLtdOg1j06A/98kPu3y2x/eS46BrSxgMvUQhBXb05xWk5mTWUO3JxaSPokYhzcE6ixLBcDk/MY9qt/VZ53uG9zQjSWRScR69TAJGLn+iYRETyATW7SxfJskp/iObgn0B0V0UB5umdc4ZKs73TPBPJjc6Iw+RwozwURcLpbG3XAYiLs48nCZLWzKAc5aRac7ubgrhu7inJgS09J+kHdKCEETveMo7nKyZOFycyRacWOwhycvqGF4D6OOo8dmTxZmKzMJsKBCmfS6wAH9wQymQgHKnJxSuV37t2jsxidXsTBSqfSRdGlg5W5aOmdVPXiHTOLIXQO+tHMdSAhmquc6Bufw1AS8+4c3BOsudKJ/ok5RVdkWY+UNuITOzGaK51YDEXQ7lVv3v1s7wTCEcEX+AQ5WJn8FC0H9wSTThY151xP9YyjMCcNZZxvT4g7KpwgAk6puA6c7h5Hipm4p0yC7CyMpmiTWQc4uCfYjsJs2DNSVNuoKoTAGc63J5QtIwW7inJUWweA6B1lvceOdCtPGJcIJhPhjorcpObd4wruRHQ/EV0loutE9IVVfv9ZIrpERB1E9BoRlclfVG2SDqpa8+7XR2YwNhNYemxkiXGw0onW/klV9nefWghyvj0Jmquc8E7MY2ByLimft25wJyIzgKcAPABgF4CHiWjXis3aADQJIWoBfBfAV+QuqJY1VzoxMDkP70RyDupGvJtvz1O4JPrWXOlEIBTBeRXO8d/SO4GIAOfbE2wpRduTnN5z8dy5HwBwXQjRI4QIAHgewLHlGwghfiaEkCLXaQBueYupbQerpIOqvrv3Uz3jKLalwZPLc3cn0v6KXJhUmnc/1T0Oq9mEfZxvT6iagmw4kpiijSe4lwDwLns9EHtvLZ8E8PJqvyCix4mohYhaRkdH4y+lxlXnZyM305q0K3a8ov3bJ3CQ8+0JZ0tPwe5imyov8Kd7JlBfakdaCufbEymaonUm7QIva4MqET0CoAnAn6/2eyHE00KIJiFEk8vlkvOjVW2pMaVnXFULJnfdmsHEbIAfx5PkYGUu2vp9qsq7++eDuDjE+fZkaa5yYtCXnBRtPMF9EIBn2Wt37L1fQET3AfgDAA8KIRblKZ5+SAe1X0V591PdYwC4f3uyNFc5EQhHVDW/+zs3ON+eTNK/czI6WMQT3M8C2E5EFURkBfAQgOPLNyCiBgDfQDSwj8hfTO17z7Zog+WbXepJR53sGkW5MwOeXO7fngwHKpxIMZPK6sAIMqxm7CuzK10UQ6guyMJnf6Uade7E/3uvG9yFECEATwB4BcBlAC8KIS4S0ZeI6MHYZn8OIAvAPxPReSI6vsbuDKsiLxOluRl446o6TuyFYBinesZxuCZf6aIYRlaqBfvLc1VTB4QQeOPqKO6syuMF0ZOEiPC7925HTRIWQ4lrhiAhxAkAJ1a89+Syn++TuVy6Q0Q4VO3Cd1sHsBgKK34yvXNjAgvBCA5VG6ftQw0OVbvwv16+gmH/PIpsyvZQ6hmbxcDkPD51qErRcrDE4BGqSXS4xoX5YBhnbyifc33j6iisFhPnWpNMelI6qYK7d+kJ4jBf4HWJg3sSNVc5YTWb8MZV5Zsl3ugawcFKJw83T7LqgiwU2dJUkZp54+oIqlyZ3OaiUxzckyjDasGBilycVLhBzTsxh57RWU7JKEBKz711fQzBcESxcswHwjhzYwKHqrnNRa84uCfZ4RoXro3MKDoF8Buxi8vhGg7uSjhc48L0YgjnFOwSebpnHIFQhOuAjnFwTzLpZFIy53ry6ig8uemozMtUrAxGdue2PFhMpOgT3MmuUaSlmHCggieM0ysO7klW5cpCiT1dsbz7YiiMt7vHcKjaxVMOKCQnLQX7yhyK5t3fuDqC5konTzmgYxzck4yIcKgmmnMNhJKfc23pncRcIIzDnGtV1OEaFy4NT2FkaiHpn907Nove8Tke46BzHNwVcLjahdlAGO/cSP5EYq9fGYHVbEJzFXeBVJJ0cX39SvKf4KTP5Hy7vnFwV8Dd1S5kWM04cWE4qZ8biQi83DmMu6vzeIV7he0sykZpbgZOXLiZ9M8+0TmMHYXZKHNym4uecXBXQFqKGffuLMCPL9xEKInd4dq8Pgz5F3C0tihpn8lWR0Q4WluEt66PYXI2kLTPHfbPo6VvEkf3ch3QOw7uCjm6txATswGcSWJq5kTnMKxmE+7dWZC0z2RrO7q3COGIwE8uJe/u/eXO6Gcd4Qu87nFwV8jhmnxkWM34UUdyUjORiMCJzmHcXe1CTlpKUj6T3d7u4hyUOTPwUmfygruUkqlyZSXtM5kyOLgrRErNvHIxOamZNq8Pw/4FHK0tTPhnsfgQEY7sTV5qRkrJfJDv2g2Bg7uCju4twsRsICnL773UMQyrxYT7OCWjKslMzSylZDjfbggc3BV0uMaFTKsZL3UmNjUTiQi8fGEYd293IZtTMqoipWaSkZ57KZaSqeSUjCFwcFdQslIzbd5JDPsX+HFchYgIR/cW4e3u8YSmZob982jllIyhcHBX2JFYaubfro8l7DP+tT2akrl3J49IVKMjsdRMIsc9vBR7MuCUjHFwcFfY+3a44My04rkz/QnZ/1wghH85N4D7dxdySkaldhfnYEdhNv7pdD+EELLvPxIR+Kcz/dhXaueUjIFwcFdYqsWMX9vvwauXbyVkGuAfnh/C9EIIjzWXyb5vJg8iwmPN5bg0PIVz/T7Z9/9W9xhujM3iseZy2ffN1IuDuwr8+h2lAIDvyHz3LoTAM6f6sKMwG41lDln3zeR1rL4Y2akWPHuqV/Z9P3OqD85MKx7Yy91gjYSDuwq4HRm4Z0cBnj/bj8VQWLb9nuufxOXhKTzWXM7T+6pcZqoFH25040TnTYzNLMq230HfPF67fAu/tt+j+KLsLLk4uKvEo81lGJsJ4McyTiT1zKk+ZKdacKy+WLZ9ssR55GAZAuEIXjjrlW2fz53pA/Du0yEzDg7uKvHebXkod2bg2VN9suxvdHoRJzqH8eFGN88AqRHb8rNwZ5UTz53pRziy9YbVxVAYz7/jxT07CuB28CLYRsPBXSVMJsIjB8vQ0jeJi0P+Le/vhbP9CIYFHjnIDala8lhzGQZ983j18q0t7+vlzpsYnw3gUW5MNyQO7iry0UYPslMt+D8/6drSfiZnA/h/P7+BQ9UubMvnrm9act/OAnhy0/GXP+3a0t17IBTB117twrb8LLx3W56MJWRawcFdRWwZKfide7fh9SsjW1o8+a9eu4bphSC+eGSHjKVjyWAxm/DFB3biys3pLeXenznVi97xOfzB0Z0wmbgx3Yg4uKvMJ+4sR5kzA3/6o0ubmpLg+sg0nj3dh4cPlGJHYU4CSsgS7YE9hThQnov/85OrmFoIbvjvx2cW8VevXcOhahfex+ukGhYHd5VJtZjx34/sxLWRGXznnY33e//Tly4jI8WMz/5KdQJKx5KBiPA/P7gLE3MBPPX69Q3//V++2oW5QBj/4+jOBJSOaQUHdxV6/64CNFc68Rc/7YJ/Pv47tzeujuCNq6P43Xu3w5mVmsASskTb67bhI/vc+Pu3etE3Phv33129OY3nzvTjkTtKsb0gO4ElZGrHwV2FpDs3/3wQn3vxPIJxpGcGJufwxe91osyZgcfu5N4RevD7H6iBxUz43efPY2YxtO72/rkgPvN8G7JSLfi9+/jJzejiCu5EdD8RXSWi60T0hVV+n0pEL8R+f4aIyuUuqNHsKs7BHx/bg1cvj+Dz3+1A5DY9J0anF/HoN9/BzGIIf/vxRh6JqBP5OWn4q4cacGHQj8efacFCcO3Ry3OBEH7z2++gZ3QWf/Pr++DItCaxpEyN1g3uRGQG8BSABwDsAvAwEe1asdknAUwKIbYB+EsAX5a7oEb06MEy/P4HavD9tkH80b9eXHXGQP98EI996x3c9C/g27+5H7uKuRFVT35lVwG++tFavN09jt/5TtuqjeyLoTA+9Wwrznt9+OuH63F3tUuBkjK1iWfo4gEA14UQPQBARM8DOAbg0rJtjgH4o9jP3wXwN0REIhHzlxrMpw9XYWo+iG+82YNrt2bwkUY3PrCnEL65AH54fggvtngx5JvHNz+xH41luUoXlyXArza4MTUfwh8ev4hf/b9v42NNbhytLUZECPxr+xBebBnA5eEpfOUjtbh/D8/XzqJovfhLRB8BcL8Q4j/GXj8K4A4hxBPLtrkQ22Yg9ro7ts2aK1A0NTWJlpYWGb6C/gkh8PSbPfjHM33wTszDajEhEIrewe0vd+CJe7bjEN+t6d6LLV588+c3cPXWNCwmggAQjgjsKsrBpw5V4lh9idJFZElARK1CiKb1tkvqpCNE9DiAxwGgtJQnMooXEeFTh6rw+N2VONc/iZc6biI3MwXH6kvgyeU5Q4ziY00efKzJg8vDUzjePgQzER6sL0Y194phq4gnuA8C8Cx77Y69t9o2A0RkAWADML5yR0KIpwE8DUTv3DdTYCMjIjSW5XL6xeB2FuVgZxG3rbDbi6e3zFkA24mogoisAB4CcHzFNscBfCL280cAvM75dsYYU866d+5CiBARPQHgFQBmAN8SQlwkoi8BaBFCHAfwTQDPEtF1ABOIXgAYY4wpJK6cuxDiBIATK957ctnPCwA+Km/RGGOMbRaPUGWMMR3i4M4YYzrEwZ0xxnSIgztjjOkQB3fGGNOhdacfSNgHE40C6Nvkn+cBWHNqAx0z4vc24ncGjPm9jfidgY1/7zIhxLrzjSgW3LeCiFrimVtBb4z4vY34nQFjfm8jfmcgcd+b0zKMMaZDHNwZY0yHtBrcn1a6AAox4vc24ncGjPm9jfidgQR9b03m3BljjN2eVu/cGWOM3Ybmgvt6i3XrARF5iOhnRHSJiC4S0Wdi7+cS0U+J6Frs/w6lyyo3IjITURsR/Sj2uiK26Pr12CLsulv5mYjsRPRdIrpCRJeJqNkgx/q/xOr3BSL6DhGl6e14E9G3iGgktlqd9N6qx5ai/jr23TuIaN9WPltTwT3Oxbr1IATgc0KIXQAOAvjt2Pf8AoDXhBDbAbwWe603nwFwednrLwP4y9ji65OILsauN38F4MdCiB0A6hD9/ro+1kRUAuB3ATQJIfYgOp34Q9Df8f42gPtXvLfWsX0AwPbYf48D+NutfLCmgjuWLdYthAgAkBbr1hUhxLAQ4lzs52lET/YSRL/rP8Q2+wcAH1KmhIlBRG4ARwH8Xew1AbgH0UXXAX1+ZxuAuxFdEwFCiIAQwgedH+sYC4D02OptGQCGobPjLYR4E9E1LpZb69geA/CMiDoNwE5Em17xXGvBvQSAd9nrgdh7ukVE5QAaAJwBUCCEGI796iaAAoWKlShfA/B5AJHYaycAnxAiFHutx+NdAWAUwN/H0lF/R0SZ0PmxFkIMAvgqgH5Eg7ofQCv0f7yBtY+trPFNa8HdUIgoC8C/APg9IcTU8t/FljHUTVcnIvoggBEhRKvSZUkyC4B9AP5WCNEAYBYrUjB6O9YAEMszH0P04lYMIBO/nL7QvUQeW60F93gW69YFIkpBNLD/kxDie7G3b0mPabH/jyhVvgS4C8CDRNSLaLrtHkRz0fbYYzugz+M9AGBACHEm9vq7iAZ7PR9rALgPwA0hxKgQIgjge4jWAb0fb2DtYytrfNNacI9nsW7Ni+WavwngshDiL5b9avlC5J8A8MNkly1RhBBfFEK4hRDliB7X14UQHwfwM0QXXQd09p0BQAhxE4CXiGpib90L4BJ0fKxj+gEcJKKMWH2Xvreuj3fMWsf2OIDHYr1mDgLwL0vfbJwQQlP/ATgCoAtAN4A/ULo8CfqO70H0Ua0DwPnYf0cQzUG/BuAagFcB5Cpd1gR9/8MAfhT7uRLAOwCuA/hnAKlKly8B37ceQEvseP8AgMMIxxrAHwO4AuACgGcBpOrteAP4DqJtCkFEn9I+udaxBUCI9gbsBtCJaE+iTX82j1BljDEd0lpahjHGWBw4uDPGmA5xcGeMMR3i4M4YYzrEwZ0xxnSIgztjjOkQB3fGGNMhDu6MMaZD/x/A6LtLH93emwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max = (100 // 5) + 1)\n",
    "\n",
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "    scheduler.step()\n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "### CyclicLR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11901b8d0>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4W+WZ9/HvI3l3vMa2nN1ZHNuywxY3JCEESGKVPYRpO22nLdN2SjstTNvpTF8gUKCFtJ2ZdrozZWCgnXZop5SEsMtZSAiEkIQtlmwndjZnsbzvm5bn/cM2A2mCHVvSkY7uz3Xlii3LOrdy7F+Obj3nPkprjRBCiOhnMboAIYQQwSGBLoQQJiGBLoQQJiGBLoQQJiGBLoQQJiGBLoQQJiGBLoQQJiGBLoQQJiGBLoQQJhEXzo3l5OTogoKCcG5SCCGi3v79+1u01rlj3S+sgV5QUMC+ffvCuUkhhIh6Sqlj47mftFyEEMIkJNCFEMIkJNCFEMIkJNCFEMIkJNCFEMIkxgx0pdR/KaWalFJV77stWylVqZQ6NPJ3VmjLFEIIMZbxHKE/Dlx9xm13AFu11oXA1pHPhRBCGGjMQNda7wTazrh5LfCbkY9/A9wU5Lpintcf4Pd7jjHg9RtdihAiSky0h27TWp8e+bgRsJ3rjkqpW5VS+5RS+5qbmye4udjzkquR9RureOrNk0aXIoSIEpN+U1QPX2X6nFea1lo/rLUu11qX5+aOeeaqGOF0eQCodDcaXIkQIlpMNNA9SqlpACN/NwWvJDHkC7C9tgmrRfFqfSs9gz6jSxJCRIGJBvpm4JaRj28Bng5OOQJgz5FWugd8fHHFXIZ8AXYelFaVEGJs41m2+ASwGyhSSp1QSn0R+AFQoZQ6BKwZ+VwESaXbQ3K8la+vLiQrJZ5Kt8fokoQQUWDMaYta60+d40urg1yLALTWOF0eVi7MITUxjtUlNpyuRrz+APFWOQ9MCHFukhAR5sDJThq7BnDY8wFw2G10Dfh448iZK0eFEOKDJNAjTKXbg9WiWFWcB8DlhbkkxVuk7SKEGJMEeoRxujx8pCCLrNQEAJITrFxemIvT1cjwClEhhDg7CfQIcqy1l1pPNxUj7ZZRFXYbpzoHcJ3qMqgyIUQ0kECPIKNtFYf9gyferimxYVHglLaLEOJDSKBHEKfLQ8m0dGZlp3zg9uzUBMoLsnG65KxRIcS5SaBHiNaeQfYda6PCfvaxOA67jZrGbhra+sJcmRAiWkigR4itNU0E9F+2W0aNBr20XYQQ5yKBHiGcLg8zMpMpnZ5+1q/PmZpKcX6aDOsSQpyTBHoE6Bvy8cqhZirsNpRS57xfhd3GG0faaO8dCmN1QohoIYEeAV451MKgL3DOdssohz2fgB5uzwghxJkk0COA0+UhIzmej8zN/tD7lc1IZ1pGkrRdhBBnJYFuMJ8/wNYaD6uK88YcvqWUosJuY8fBZvqH5NJ0QogPkkA32N6j7XT0ecdst4xy2PMZ8AbYVdcS4sqEENFGAt1glW4PCXEWVi4c3+X5Lp2XTVpSnLRdhBB/QQLdQFprnO5GViwYnn0+HvFWC6uK89hS3YQ/IMO6hBD/RwLdQNWnuznR3j/udssohz2ftt4h9h9rD1FlQohoJIFuoEq3B6Vgdcn5BfoVRbkkWC3SdhFCfIAEuoGc7kYumZ1FblrieX3flMQ4li+YitPtkRnpQoj3SKAb5GRHP65TXefdbhnlsOdzrLWPQ009Qa5MCBGtJNANUjkyCvdc0xXHsqZk+BJ1MlJXCDFKAt0gTreHBXlTmJc7ZULfn5eexMWzM2X6ohDiPRLoBujs87LnSNuE2y2jHPZ83j3RyenO/iBVJoSIZhLoBthW68Ef0BNut4wa/f5KOUoXQiCBbginy0NeWiIXzsyc1OMMt2xSJdCFEIAEetgNeP3sONjMGrsNi+Xcs8/Hq8JuY3d9K5393iBUJ4SIZhLoYfZafQt9Q/5J989HOez5+AKal2tlRroQsU4CPcycLg9TEuNYNn9qUB7v4lmZ5ExJlNUuQggJ9HDyBzRbqj1cUZRLYpw1KI9psSgq7HnsqG1m0Ccz0oWIZRLoYfR2QzstPUNBa7eMctjz6Rn0sbu+NaiPK4SILhLoYeR0e4i3Kq4qzgvq4y6bP5XUBKusdhEixkmgh4nWGqfLw9J5U0lPig/qYyfFW7miKJdKt4eAzEgXImZJoIdJfXMPR1p6g95uGeWw59PUPcg7JzpC8vhCiMgngR4mo6tQ1oQo0K8qyiPOomS1ixAxTAI9TJwuDxfMzGBaRnJIHj8jJZ5L52VLH12IGDapQFdKfVMp5VJKVSmlnlBKJQWrMDPxdA3wdkNHyNotoxz2fOqaejjcLDPShYhFEw50pdQM4B+Acq11GWAFPhmswsxkS/XwUXOFPT+k21kjw7qEiGmTbbnEAclKqTggBTg1+ZLMx+nyMGdqCgttE5t9Pl4zMpMpm5EufXQhYtSEA11rfRL4N+A4cBro1Fo7z7yfUupWpdQ+pdS+5ubmiVcapboHvOyub8Vht6HU5IdxjcVhz+fN4+00dw+GfFtCiMgymZZLFrAWmAtMB1KVUp85835a64e11uVa6/Lc3NyJVxqldhxsZsgfCHm7ZVSF3YbWsLVajtKFiDWTabmsAY5orZu11l7gKWB5cMoyD6fLQ3ZqAovnZIVle8X5aczKTpa2ixAxaDKBfhxYqpRKUcO9hNVAdXDKMochX4DttU2sKcnDGoTZ5+OhlMJhz2dXXQu9g76wbFMIERkm00PfAzwJvAkcGHmsh4NUlynsOdJK94AvbO2WURV2G0O+ADsPxt57FkLEskmtctFa36u1LtZal2mtP6u1lnfi3sfp8pAUb2HFgpywbrd8ThZZKfHSdhEixsiZoiGitabS7WFlYS7JCcGZfT5ecVYLq4ptbKtpwusPhHXbQgjjSKCHyIGTnTR2DeAoDW+7ZZSj1EZnv5e9R9oM2b4QIvwk0EOk0u3BomB1kGefj9fKwlyS4i3SdhEihkigh4jT5eEjBdlkpSYYsv3kBCsrFgzPSNdaZqQLEQsk0EPgWGsvtZ5uw9otoxylNk529OM61WVoHUKI8JBAD4HR4Vihnq44ltXFeViUDOsSIlZIoIeA0+UZOWMzxdA6pk5JpHxOtvTRhYgREuhB1tozyL5jbYa3W0Y5Sm1Un+6ioa3P6FKEECEmgR5kW2uaCGjj2y2jKmRGuhAxQwI9yJwuD9Mzkiidnm50KQDMmZpKkS0Np7vR6FKEECEmgR5E/UN+dtU1UxGm2efjVWG3sfdoO+29Q0aXIoQIIQn0INp5qJkBbyBi+uejHKU2/AHNtpomo0sRQoSQBHoQOV0e0pPiWDI32+hSPmDRjAzy05Ok7SKEyUmgB4nPH2BbjYdVxXnEWyPrn1UpRYXdxs6DLQx4/UaXI4QIkchKnii271g77X3eiGu3jHKU2uj3+tl1qMXoUoQQISKBHiSVbg8JcRZWLozM66ZeOncqaUlx0nYRwsQk0INAa43T3chl86cyJTHO6HLOKiHOwlVFeWypbsIfkGFdQpiRBHoQ1DR209DWH7HtllGOUhttvUPsP9ZudClCiBCQQA+CSrcHpWB1iTGzz8frioW5JFgtVErbRQhTkkAPAqe7kYtnZZKXlmR0KR8qLSmeZfOn8pJLZqQLYUYS6JN0sqOfqpNdEd9uGeUotXG8rY+Dnh6jSxFCBJkE+iRtGRl6VREhw7jGsqZkdFiXtF2EMBsJ9ElyuhuZn5vK/NwpRpcyLrb0JC6alSkz0oUwIQn0Sejs8/L64ciZfT5ejlIb757o5HRnv9GlCCGCSAJ9ErbXDq/pjpZ2y6jRWe1b5ChdCFORQJ8Ep7uR3LRELpqZaXQp52V+7hTm5aRK20UIk5FAn6ABr58dtcOzzy2WyJl9Ph5KKSpKbeyub6Wz32t0OUKIIJFAn6Dd9a30Dvmjrt0yymG34QtoXq6VGelCmIUE+gQ53Y2kJlhZPn+q0aVMyEWzssiZkihtFyFMRAJ9AgIBTaW7iSuL8kiMsxpdzoRYLYoKex47apsZ9MmMdCHMQAJ9At5q6KClZxBHaXS2W0ZV2G30DPrYXd9qdClCiCCQQJ8Ap7uROIviyqLIHsY1luXzc0hJsErbRQiTkECfgEq3h6XzppKRHG90KZOSFG/lioW5bHF7CMiMdCGingT6eapr6uFwc2/Ut1tGOUptNHUP8s6JDqNLEUJM0qQCXSmVqZR6UilVo5SqVkotC1ZhkWr0Em6jQ66i3aoiG1aLkraLECYw2SP0nwIvaq2LgQuB6smXFNkq3R4Wzchgemay0aUERUZKPJfOzaZSAl2IqDfhQFdKZQArgUcBtNZDWmtTv25v6hrgreMd781CMQuH3TbSSpIZ6UJEs8kcoc8FmoHHlFJvKaUeUUqlBqmuiLSlevisymibrjiWipHnI0fpQkS3yQR6HHAJ8JDW+mKgF7jjzDsppW5VSu1TSu1rbm6exOaM53Q3Mjs7hYW26Jh9Pl4zMpMpnZ7OSy656IUQ0WwygX4COKG13jPy+ZMMB/wHaK0f1lqXa63Lc3NzJ7E5Y/UM+nitrpUKuw2lomsY13g47Pm81dBBc/eg0aUIISZowoGutW4EGpRSRSM3rQbcQakqAu2obWbIHzBd/3xUhd2G1rC1WtouQkSrya5yuR34vVLqXeAiYMPkS4pMTncj2akJLJ6TZXQpIVEyLY2ZWcmyfFGIKBY3mW/WWr8NlAeplojl9QfYVtPE1aX5xFnNeS6WUgqHPZ/f7TlG76CP1MRJ/WgIIQxgznQKsj2H2+ge8EXt7PPxqrDbGPIF2Hkwut+8FiJWSaCPg9PdSFK8hcsLo/dN3fH4SEEWmSnx0nYRIkpJoI9Ba02l28PlhbkkJ0Tn7PPxirNaWFWcx7aaJrz+gNHlCCHOkwT6GKpOdnG6c8C0q1vO5LDn09nvZe+RNqNLEUKcJwn0MTjdjVgUrDbJMK6xrFyYQ2KcRdouQkQhCfQxVLo9lBdkk52aYHQpYZGSEMflhTlUuj1oLTPSw6Gtd4jvv1BNS4+c1CUmRwL9Qxxr7aWmsTtm2i2jHPZ8Tnb04zrVZXQpMeG7z7j49Y7D3P+Mac/LE2Eigf4hRodVOezmGsY1llUleSiFtF3CYNehFja9fYp5Oak8884pWTIqJkUC/UM43R6K89OYPTXF6FLCKmdKIuVzsmT6YogNeP3cvekABVNT2HTbZczLSeXuTVUMeP1GlyailAT6ObT1DrHvaFvMtVtGOez5VJ/uoqGtz+hSTOtX2+s42trHAzctIj0pngfWlXG8rY+fbztkdGkiSkmgn8PWag8Bbb7Z5+M1elasHKWHRl1TNw/tqOemi6azojAHgOXzc7j5khk8vPMwhzzdBlcoopEE+jk43R6mZyRROj3d6FIMUZCTykLblPeuoSqCR2vN+o1VJMdbWX+d/QNfW39tCamJcazfWEUgIKuMxPmRQD+L/iE/rxxqNu3s8/Fy2PPZe7Sd9t4ho0sxlSf3n2DPkTbuvLaE3LTED3xt6pRE7rymmDeOtvHk/hMGVSiilQT6WbxyqJkBbyBm2y2jKuw2/AHNtpomo0sxjbbeITY8X035nCz+unzWWe/z8cWzWFKQzYYXqmmVteniPEign0Wl20N6UhxL5mYbXYqhFs3IID89SfroQbTh+Wq6B3w8uG4RFsvZX/1ZLIoH15XRO+jjweerw1yhiGYS6Gfw+QNsqfawqjiPeJPOPh8vi0Wxxp7HjoPNspQuCF4/3MqT+0/wpZXzKMpP+9D7FtrSuHXlPJ568ySv1bWEqUIR7WI7sc5i/7F22vu8Md9uGeWw59Pv9bPrkITKZAz6/Ny18QCzspP5h1WF4/qe21cVMjs7hbs3VTHok/9Qxdgk0M9Q6faQYLWwcqG5Z5+P19J5U0lLjJO2yyT9esdhDjf38r21ZeMew5wUb+WBm8o43NLLQy/Xh7hCYQYS6O+jtcbp9nDZgqlMkUuwAZAQZ+HK4jy2VHvwyzK6CTnS0ssvttdx3QXTuLIo77y+d+XCXG64cDq/2l7P4eaeEFUozEIC/X1qPd0cb+uTdssZHHYbrb1DvHW83ehSoo7Wmns2VZFotXDv9faxv+Es7rm+hMR4C3dvqpIJmOJDSaC/T6XLg1KwuuT8jqLM7sqiXOKtipdccpLR+Xr67VPsqmvh21cXkZeeNKHHyEtL4v9dXcxr9a1sfOtkkCsUZiKB/j5Ot4eLZ2WSlzaxXzyzSkuKZ9l8mZF+vjr7vDzwnJsLZ2Xy6UvnTOqxPr1kNhfPzuTB56rp6JMTvcTZSaCPONXRz4GTnVTE2Kjc8XLYbRxt7aOuSfq44/WDF2to7/OyYV0Z1nOsOR8vi0WxYd0iOvq9/OCFmiBVKMxGAn3EluqR2eelsTldcSyjw7pkRvr47D/WxhNvHOfzywsonZ4RlMcsmZbOF1fM5Q97G9h7VK75Kv6SBPoIp8vD/NxU5udOMbqUiGRLT+LCWZk4pY8+Jq8/wF1PVTE9I4lvViwM6mN/Y00hMzKTueupAwz5AkF9bBH9JNCBzn4vrx9ulXbLGBx2G++c6OR0Z7/RpUS0R145Qq2nm/vXlpEa5OWvKQlxfHdtKYeaevjPVw4H9bFF9JNAB16ubcIX0NJuGcNHR/59tlTLsK5zaWjr46dbD+Kw295rUwXb6hIbV5fm87OthzjeKhcgEf9HAp3hdktuWiIXzcw0upSINj93CnNzUqXtcg5aa77zdBVWpbjvxtKQbuveG+3EWRT3PC1r08X/iflAH/T5ebm2iTUltnNOvxPDlFI47DZeP9xK14DX6HIizgtVjWyvbeYfHUVMz0wO6bamZSTzTx8tYsfBZp47cDqk2xLRI+YD/bX6VnqH/NJuGSdHqQ2vX/NyrVyd/v26Brzct9lF6fR0blk2uTXn4/W5ZQUsmpHB/c+46eyX/2CFBDpOl4fUBCvL5081upSocNGsLHKmJErb5Qw/eqmW5p5BNqxbRFyYxi5bR9amt/YM8m8v1YZlmyKyxXSgBwKaLdUerizKIzFufBPwYp3VolhTksfLtc0y0nXEOw0d/Pb1Y3xu6RwunBXe92EWzczgc8sK+N2eY7zd0BHWbYvIE9OB/vaJDpq7B6Xdcp4cpTZ6Bn28flhObvH5A9y18QB5aYl866NFhtTwLcdCbGlJ3PXUAXx+WZsey2I60CvdHuIs6rxHmsa65fNzSEmwStsF+M3uY7hOdXHvDaWkJ8UbUkNaUjz33WjHfbqLx187akgNIjLEdKA7XY0snTeVjGRjfhGjVVK8lSsW5rKl2kMghmekn+ro50fOWq4qyuWaMmNPSvtoaT6ri/P4kfMgJzvkxK9YNelAV0pZlVJvKaWeDUZB4VLf3EN9c6+0Wyaowm7D0zXIuyc7jS7FMPdtdhHQmu+uLUMpY5e8KqW4f+3w2vd7n3YZWoswTjCO0L8ORN2lyUcvqbamRAJ9IlYV52G1qJhtuzhdjTjdHr6+eiGzslOMLgeAmVkpfGNNIVuqPTK7PkZNKtCVUjOB64BHglNO+DhdjSyakRHyE0DMKjMlgSUF2TF5rdHeQR/3bXZRZEvj7y6fa3Q5H/CFFXMpzk/j3qdd9Az6jC5HhNlkj9B/AnwbiKq31pu6B3iroQNHiGZtxApHqY1DTT0xd63Lf688yKnOATbcXEZ8mNacj1e81cKGmxfh6R7gx86DRpcjwmzCP41KqeuBJq31/jHud6tSap9Sal9zc2ScXbi1ugmtoUL655MyOnwqlo7SXac6eey1o3xqyWwWz8k2upyzumR2Fp9eMpvHXztCVQy/xxGLJnN4cRlwo1LqKPAHYJVS6ndn3klr/bDWulxrXZ6bmzuJzQWP09XI7OwUimxpRpcS1WZmpWCflh4zge4PaO7aWEVWSjx3XF1sdDkf6ttXF5OdmshdGw/gj+GVSLFmwoGutb5Taz1Ta10AfBLYprX+TNAqC5GeQR+v1rdSYbcZvjLBDBylNvYfb6e5e9DoUkLu93uO8U5DB3dfZycjJbKXumYkx/OdG+y8e6KT/9591OhyRJhEVgMwDHYebGbIF5D+eZA47PloDdtqzH2U7uka4F9frGXFghzWXjTd6HLG5YYLpnF5YQ7/5jxIY+eA0eWIMAhKoGutX9ZaXx+Mxwo1p6uR7NQEFs/JMroUUyiZlsaMzGScLnMH+nefdTPoD/DATcavOR8vpRQP3FSG1x/gu8/K2vRYEFNH6F5/gG01TawqzgvbRDyzU0rhKLXxSl0LvSZdJre9tonn3j3NbVctoCAn1ehyzsucqancvmoBzx9oNP2rKBFjgf7GkTa6BnwhuzRYrKqw2xjyBdh5MDJWMQVT/5Cf7zxdxfzcVL58xTyjy5mQW1fOZ0HeFO7Z5KJvyJz/6YphMRXoTlcjSfEWVhZGxmobs1hSkE1GcrwpV7v8bNshGtr6eXDdoqgdsZwQZ2HDukWc7Ojnp1sPGV2OCKGYCXStNZVuD5cX5pKcEJ2/mJEqzmphdUkeW2ua8JpofGttYzf/ufMwH1s8k6XzovsCKEvmZvOJ8pk8+soRahq7jC5HhEjMBLrrVBenOgdkdUuIOOw2Ovu97D1qjhnpgYBm/cYDpCXFcde1JUaXExR3XlNCenI8dz11IKanZJpZzAS609WIRcFqGcYVEisX5pIYZzHNapc/7mtg37F27rq2hOzUBKPLCYqs1ATWX1vCm8c7eGLvcaPLESEQO4Hu9lBekG2aX85Ik5IQx4oFOVS6PWgd3Ud/LT2D/OCFGi6dm83HFs80upyguvmSGSybN5UfvFATEyeDxZqYCPSGtj5qGrul3RJijlIbJzv6cZ+O7h7tg89V0zfk48F1i6Jmzfl4KaV4YF0Zg94ADzznNrocEWQxEejOkdUXslwxtFaX2FCKqG677DrUwsa3TvKVK4aX+pnR/NwpfOXK+Tz99ilTLjWNZbER6K5GimxpzJkaXSeFRJucKYksnp0VtcsXB7x+7nm6ioKpKXztqgVGlxNSX71yPnNzUrnn6SoGvH6jyxFBYvpAb+sdYu/RNrnUXJg4Sm24T3fR0NZndCnn7Vcv13OkpZcHblpEUry5l7YmxVt58KYyjrX28cvtdUaXI4LE9IG+raaJgJZ2S7hU2IcvlhxtR+l1TT38x8v1rL1oOisKc4wuJyyWL8hh3cUz+I8d9dQ1dRtdjggC0we609XItIwkFs3IMLqUmDA3J5XCvClRFehaD685T4q3cPd1dqPLCav115WQkhDHXRuron51UiQL17p/Uwd6/5CfnYeaZfZ5mDlKbbxxtI2OviGjSxmXP795kj1H2rjjmhJy0xKNLiescqYkcuc1xbxxpI0/7T9hdDmmtONgM1f/dCdNXaEfYWzqQN9V18KANyDtljCrsOfjD2i21TQZXcqY2nuH2PB8NYvnZPHJj8wyuhxDfKJ8FuVzsvj+89W09UbHf8LRQGvNQy/X8/nH3sBqsTAUhrEYpg50p6uRtKQ4Lp0b3XM4os0FMzKwpSdGxfLF779QTVe/lwfXlWGxxOarOItFseHmRXQP+NjwfLXR5ZhC35CP2554ix++WMO1i6bx579fxsyslJBv17SB7vMH2FLtYVVxHglxpn2aEcliUVTYbew42BzRS+L2HG7lf/ed4IuXz6U4P93ocgy10JbGrSvn8eT+E+yubzW6nKjW0NbHzb96jRcOnOaOa4r5+acuJiUhLizbNm3S7T/WTnufV9otBqmw59Pv9bPrUIvRpZzVkC/A+k1VzMxK5hurFxpdTkS4fVUhs7KTWb/pAIO+yP2POJK9WtfCDb/YxamOfh77/BK+csX8sL5/Z9pAr3R7SLBauGKhzD43wrJ5U0lLjIvY1S4P76ynrqmH760tk3HKI5ITrHxvbRmHm3v59Y7DRpcTVbTWPPLKYT776B7y0hLZfNsKQ7LHlIGutcbp9rB8wVTSkiL76uxmlRBn4criPLZUe/BH2KjWoy29/HxbHdcuyueq4jyjy4koVxblcd0F0/jF9jqOtPQaXU5UGPD6+eYf3+aB56px2PPZ+NXLDLtUoSkD/aCnh+NtfdJuMViF3UZr7xBvHW83upT3aK255+kq4q0W7r2h1OhyItK919tJtFq4e9MBWZs+hpMd/XzsP17j6XdO8U+OhTz0mUtITQxPv/xsTBnoTlcjABUy+9xQVxblEm9V7w1HiwSb3znFK4da+OePFmFLTzK6nIiUl57Et68u4tW6Vp5++5TR5USs3fWt3PDzXRxr6ePRW8q5bVWh4ee7mDPQ3R4unp1JnvzCGio9KZ5l83Nwuhoj4kivs8/L956t5oKZGXxm6Ryjy4lon750DhfOyuSB59xRc4JYuGitefzVI3zm0T1kpybw9G2Xsao4Mg4eTRfopzr6OXCyU9otEaLCbuNoax91TT1Gl8IPX6qhrXeQDesWYY3RNefjZbUoNqwro73Pyw9frDG6nIgx4PXzz0++y33PuLmqKI+NX13OvNzIGbNsukDfUj388l4uZhEZRtteRrdd9h9r53/2HOfzl82lTOb6jEvp9Ay+cFkBT7zRwD6TXCt2Mk539vPXv97Nk/tP8PXVhTz82cURt+jCdIFe6fYwLzeVBXlpRpcigPyMJC6clWlooHv9AdZvPMC0jCT+sULWnJ+Pb6xZyPSMJNZvrMIbhlPXI9Xeo23c8PNd1DX18PBnF/PNioUReWaxqQK9s9/L7vpWabdEGIfdxjsNHTR2hn440dn8164j1DR2c9+NpYauQIhGqYlx3L+2jFpPN4+8csTocsJOa83vXj/Gpx5+nbSkeDZ97TIcpflGl3VOpgr0l2ub8AU0Dnvk/oPHotH2V2V1+I/SG9r6+MmWQ1TYbXw0gn8RI9nwv52Nn249GJUXLpmoQZ+fuzYe4O5NVVxemMOmr11GoS2yX/mbKtCdbg85UxK5eFam0aWI91mQN4W5OalhP2tUa829m10oBfffKGvOJ+O+G0uxKsU9T8fG3PSmrgE+9fDrPPFGA7ddtYBHbvkIGcmR1S8/G9ME+qDPz8s1TVTY8yKytxXLlBoe1rWB72OQAAAPUklEQVS7voWuAW/YtvtiVSPbapr4x4qFTM9MDtt2zWhaRjL/6Cji5dpmnj/QaHQ5IbX/WDvX/3wXNY3d/OpvLuGfPloUNauiTBPou+tb6R3yS7slQjnsNrx+zY7a8FxlvnvAy33PuLBPS+dvlxeEZZtmd8uyOZROT+f+Z1xh/Y85nP7wxnE++fBukuKtPPXV5Vy7aJrRJZ0X0wS60+0hNcHKsvky+zwSXTw7i5wpCWFb7fIj50GaugfZcPMi4qym+TE3VJzVwvdvXkRLzyA/eqnW6HKCasgX4O5NB7jjqQMsnTeVzbddFpUjlU3xkx4IaCrdHq4oyjX91dqjldWiWF1s4+WaJoZ8oV3+9u6JDn67+yifXTqHi+T9lKC6YGYmn1tWwG9fP8Y7DR1GlxMUzd2D/M0jr/O714/z5Svm8fjnl5CZkmB0WRNiikB/50QHzd2D0m6JcI5SG92DPl4/HLoLKPj8Ae7aeICcKYn800eLQradWPYtx0Ly0hK586kD+KJ8bfo7DR3c+ItdHDjZyc8+dTF3XlMSNf3yszFFoDvdHuIsiquKZBRqJLtsQQ4pCVac7tC9qfbb3ceoOtnFd26wkx5hZ/GZRVpSPPfeUIr7dBePv3bU6HIm7E/7Gvj4r3djtSj+/PfLufHC6UaXNGkTDnSl1Cyl1HallFsp5VJKfT2YhZ0Pp6uRS+dlk5Eiv8CRLCneysrCXCrdHgIhmJF+urOfHzlrubIol+ui7M2saHNNWT6rivP4ceVBTnX0G13OefH6A9y32cU/P/ku5XOy2HzbCkqnm2McxGSO0H3At7TWdmAp8DWllD04ZY1ffXMP9c290m6JEo5SG56uQQ6c7Az6Y9+/2Y1fa763tszwMaZmp5Ti/htL0Rru2+wyupxxa+0Z5DOP7OHx147yhcvm8tsvLCE7NTr75Wcz4UDXWp/WWr858nE3UA3MCFZh4zV6ssoaOd0/KqwqzsNqUbzkCm7bZYvbw4uuRv5hdSGzskN/dXUBs7JT+PqaQpxuz3vXIIhkVSc7ufEXr/J2Qwc//sSFfOcGu+lWQAXl2SilCoCLgT3BeLzz4XQ1UjYjnRly4khUyExJYElBdlDPGu0b8nHvZhcLbVP40uXzgva4YmxfXDGX4vw07tvsonfQZ3Q557TprZP81UOvobXmya8s5+ZLZhpdUkhMOtCVUlOAPwPf0Fp3neXrtyql9iml9jU3B/ekkqbuAd5q6JB2S5RxlNo41NQTtGtW/mTLIU529LNh3SLiTXbEFenirRYeXFfGqc4B/r3yoNHl/AWfP8ADz7r5xh/f5sJZmWy+fQWLZpqjX342k/rpV0rFMxzmv9daP3W2+2itH9Zal2uty3Nzg3sV7K3VTWiNTFeMMqP7qzIIq13cp7p4dNcRPvmRWZQXZE/68cT5Wzwnm08tmc1jrx2lKgTvjUxUe+8Qf/vYXh7ZdYRbls3h9393KTlTEo0uK6Qms8pFAY8C1VrrHwevpPGrdHuYlZ1McX5kT0ATHzQzKwX7tPRJt138Ac1dGw+QmRzPHdcUB6k6MRF3XF1MVko86zcewB+CFUzny32qixt+sYs3jrTxLx+7gPvXlsXEq7fJPMPLgM8Cq5RSb4/8uTZIdY2pZ9DHrroWKkryZUVDFKqw29h3rJ2WnsEJP8b/vHGctxs6uPv6kqg9s88sMlLiued6O++c6OT3e44ZWssz75zi5odexesP8McvL+UT5bMMrSecJrPKZZfWWmmtL9BaXzTy5/lgFvdhdh5sZsgXwFEq7ZZo5Ci1oTVsneCM9KbuAf7lxRouWzCVmy4K++IqcRY3XjidFQty+NcXa/F0hf9iJv6A5gcv1HD7E29RNj2DZ25fwcWzs8Jeh5Gi9jVIpdtDZko85XNia4eZhX3a8MqkibZdvvdsNYPegKw5jyBKKR64qYxBf4DvPusO67Y7+7x8/vG9/MeOej596Wz+50tLyUtLCmsNkSAqA93rD7C12sPqYpvp1pHGitEZ6a8caqFv6PyWu+042Mwz75zia1ctiKgrrgsoyEnl9qsW8Ny7p9le2xSWbR70dHPjL3exu76FDesWsWHdIhLiYjMXovJZ7z3SRteAT9otUc5RamPQF2DnwZZxf8+A1889m6qYl5PKV66UNeeR6NYr5jE/N5V7NlXRP+QP6bZerDrNTb98lb4hP3+4dSmfvnR2SLcX6aIy0J1uD0nxFlYWBncZpAivJQXZZCTHn9ewrp9vO8Txtj4eWFdGYpyMSo5EiXFWHly3iBPt/fxs26GQbCMQ0PzIWctXfvcmC21pPHv7ChbPkWWrURfoWg/PPl+xIJfkBPmFjmZxVguri/PYVtM0rjGsBz3d/HrHYf7qkpksn58ThgrFRC2dN5WPL57Jf+48TG1jd1Afu2vAy9/9dh8/31bHJ8pn8scvL8WWHnv98rOJukB3neriZEe/tFtMwlFqo6PPy96j7R96v0BAs37jAaYkxbH+upIwVScm485rS0hLiuOujQeCNl2zrqmbm37xKjsPNvO9taX88K8ukFdq7xN1ge50e7AoWF0ss8/NYOXCXBLjLGO2Xf60v4G9R9u565oSU03HM7Ps1ATuuraE/cfa+eO+hkk/XqXbw02/fI2uAS//86WlfHZZgaxwOkPUBXql20P5nGymmvwU3liRkhDHigU5VLo9aH32o7iWnkE2PF/DkoJsPrbYnEOVzOpji2dy6dxsvv98Nc3dEzuJLBDQ/HTLIb70233My01l820rWDJX+uVnE1WB3tDWR/XpLmm3mIyj1MaJ9n6qT5+917rhuWr6hnw8uK4MSxRfHiwWKaV4cN0i+r1+Hnzu/Nem9wz6+Mrv9vPvWw5y8yUz+N8vL2O6TFY9p6gK9NErxsswLnNZXWJDKc7adnmtroWn3jrJl1fOp9AmM3ui0YK8Kfz9FfPZ9PYpdh0a/xLVw8093PTLV9la08R3rrfzo49fKBeBH0NUBXqlu5EiWxpzpqYaXYoIopwpiSyenfUXZ40O+vzcvamKOVNTuG3VAoOqE8Hw1asWUDA1hXuermLAO/ba9O01Taz95au09gzy319cwhdWzJV++ThETaC39w7xxpE2abeYlKPUhutUFyfa+9677aGX6znc0sv31pbJkVmUS4q38sBNizjS0suvtted835aa365vY4v/GYvs7NTeOb2FbJE9TxETaBvrWkioJGLWZhUxch+3TJylF7f3MOvttdzw4XTWblQTiAzgxWFOdx00XQe2lFPXVPPX3y9d9DHV3//Jv/6Ui03XDCdJ7+ynJlZcjnB8xE1gV7pbmRaRhJlM9KNLkWEwNycVArzpuAcWe1y98YqEuMt3HO9rDk3k/XX2UmOt7J+44EPrGo61trLzb96jZdcjay/toSffvIiOXFwAqIi0PuH/Ow42EyF3SZ9NBOrsNvYc6SNx149yu7Drfy/q4tjcmKemeWmJXLHNSXsOdLGn988CQwPW7vxF6/S2DXAb76whC+tnCe/5xMUFYG+q66FAW9A2i0m5yjNxx/QfPdZNxfPzuTTS2J70JJZffIjs1g8J4sHn3Pzky0H+fxjbzAtI4nNt13G5TKfaVKiItAr3Y2kJcVx6Tw5mcDMLpiRQV5aIlaLYsO6RbLm3KQsFsWD68roHvDxky2HuLosnz///XJZvRYEcUYXMB4FOal8ZumcmLgmYCyzWBTfucFO/5CfkmnyXomZFeen8y8fu4DeQR+fWTpHWixBos51unUolJeX63379oVte0IIYQZKqf1a6/Kx7ieHvEIIYRIS6EIIYRIS6EIIYRIS6EIIYRIS6EIIYRIS6EIIYRIS6EIIYRIS6EIIYRJhPbFIKdUMHJvgt+cA47/ciXnE4vOOxecMsfm85TmPzxyt9ZiDbsIa6JOhlNo3njOlzCYWn3csPmeIzectzzm4pOUihBAmIYEuhBAmEU2B/rDRBRgkFp93LD5niM3nLc85iKKmhy6EEOLDRdMRuhBCiA8RFYGulLpaKVWrlKpTSt1hdD2hoJSapZTarpRyK6VcSqmvj9yerZSqVEodGvk7y+hag00pZVVKvaWUenbk87lKqT0j+/uPSqkEo2sMNqVUplLqSaVUjVKqWim1zOz7Win1zZGf7Sql1BNKqSQz7mul1H8ppZqUUlXvu+2s+1YN+9nI839XKXXJZLYd8YGulLICvwSuAezAp5RSdmOrCgkf8C2ttR1YCnxt5HneAWzVWhcCW0c+N5uvA9Xv+/yHwL9rrRcA7cAXDakqtH4KvKi1LgYuZPj5m3ZfK6VmAP8AlGutywAr8EnMua8fB64+47Zz7dtrgMKRP7cCD01mwxEf6MASoE5rfVhrPQT8AVhrcE1Bp7U+rbV+c+TjboZ/wWcw/Fx/M3K33wA3GVNhaCilZgLXAY+MfK6AVcCTI3cx43POAFYCjwJorYe01h2YfF8zfMnLZKVUHJACnMaE+1prvRNoO+Pmc+3btcBv9bDXgUyl1LSJbjsaAn0G0PC+z0+M3GZaSqkC4GJgD2DTWp8e+VIjYDOorFD5CfBtIDDy+VSgQ2vtG/ncjPt7LtAMPDbSanpEKZWKife11vok8G/AcYaDvBPYj/n39ahz7dug5ls0BHpMUUpNAf4MfENr3fX+r+nhJUmmWZaklLoeaNJa7ze6ljCLAy4BHtJaXwz0ckZ7xYT7Oovho9G5wHQglb9sS8SEUO7baAj0k8Cs930+c+Q201FKxTMc5r/XWj81crNn9CXYyN9NRtUXApcBNyqljjLcSlvFcG85c+RlOZhzf58ATmit94x8/iTDAW/mfb0GOKK1btZae4GnGN7/Zt/Xo861b4Oab9EQ6HuBwpF3wxMYfiNls8E1Bd1I7/hRoFpr/eP3fWkzcMvIx7cAT4e7tlDRWt+ptZ6ptS5geL9u01r/DbAd+NjI3Uz1nAG01o1Ag1KqaOSm1YAbE+9rhlstS5VSKSM/66PP2dT7+n3OtW83A58bWe2yFOh8X2vm/GmtI/4PcC1wEKgH1htdT4ie4wqGX4a9C7w98udahnvKW4FDwBYg2+haQ/T8rwSeHfl4HvAGUAf8CUg0ur4QPN+LgH0j+3sTkGX2fQ3cD9QAVcB/A4lm3NfAEwy/T+Bl+NXYF8+1bwHF8Cq+euAAw6uAJrxtOVNUCCFMIhpaLkIIIcZBAl0IIUxCAl0IIUxCAl0IIUxCAl0IIUxCAl0IIUxCAl0IIUxCAl0IIUzi/wO0D9yHnQac2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, \n",
    "                                              base_lr=0.1,  max_lr=10, \n",
    "                                              step_size_up=20, step_size_down=None, \n",
    "                                              mode='triangular2', gamma=0.1, \n",
    "                                              scale_fn=None, scale_mode='cycle',\n",
    "                                              cycle_momentum=True, base_momentum=0.8,\n",
    "                                              max_momentum=0.9, last_epoch=-1)\n",
    "\n",
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "    scheduler.step()\n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "### CosineAnnealingWarmRestarts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11982e350>]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXd4HNW5/z/vrrosrXpv7gU32XIHQnFCSyChxU6oIZBGAjeFm9t4ctN+6Z1LQugEMCWQOISWAKHZGMsd9yZbcpeL3C3t7vn9sbtCGNlaSTM7M2fP53n8WLs72jmrs+c777ztiFIKg8FgMOiFz+kBGAwGg8F6jLgbDAaDhhhxNxgMBg0x4m4wGAwaYsTdYDAYNMSIu8FgMGiIEXeDwWDQECPuBoPBoCFG3A0Gg0FDUpw6cVFRkaqrq3Pq9AaDweBJFi1a1KqUKu7pOMfEva6ujsbGRqdObzAYDJ5ERLbEc5xxyxgMBoOGGHE3GAwGDTHibjAYDBpixN1gMBg0xIi7wWAwaEiP4i4i94vIbhF57xSvi4j8RkQ2iMhyEZlg/TANBoPB0BvisdwfBC48zesXAUOj/24B7u7/sAwGg8HQH3rMc1dKvSEidac55DLgYRXZr+8dEckTkXKl1A6LxvgB1u06xLNLtnHHBcMRETtOcUre29bG+t2HuOCMMrLSHCsRSCgdoTAPz99Cql8oyUmnLJDJqPJc0lKMR083TgRDPDSviRMdYVJTfKSn+MjNSCU/O5X8rDTKA5mU5KTj8yV23Rn6hhUKVQk0d3ncEn3uQ+IuIrcQse6pqanp08neWt/K3f/ayMjyXC4dV9Gn9+grP395La+t3UNO+kouHV/BjTPqGFKSk9AxJJrlLW1877lVH3huQHoKZw0t4rwRJVw8ppzs9OS40OnOkq0H+OHza057TJrfR0VeBoOLBzCsLIcRZTmMrgwwqCg74caW4fQkdFUqpe4B7gFoaGjo087c10+v469Lt/Hdv63k7KFF5GWlWTrG03EiGKauMIsJtfk8vaiFvyzZxtNfms7I8tyEjSHRtAfDAPzuM/XUFWbTvO8ob6xv5dU1u3jhvZ388PnVfP6sQVw3rZacjFSHR2voD7G5nnPLVMZWBTjREabtWAf7j7az70g729uO07L/KC37jrFh92FeX7eHYDiyjPOyUqmvzmPywELOGlrEqPJcY+E7jBXivg2o7vK4KvqcLfh9wv+7fCyf+N1b/L/n1/DjK8fadaoPEQwrSnMz+MXV4/nWBcP51F3zuOnBhfzlKzMoyc1I2DgSSSi6eEtyMhhdGWB0ZYCLxpSj1GgWbdnPXa9t4KcvreWeNzbxjY8N45optWZRe5TYXKen+MhKSyErDfKz06gju9vj24NhNrUeZlnzARZvOcCirft5be0afvwiFGSn8ZFhxVxwRhkfGVZMZpo/kR/FgDXiPhe4VUTmAFOANrv87TFGVeRy81mD+P3rG/lkfSXTBhfaebpOQmFFRmrE11weyOTe6xu46vfzufnhRubcMk3LL3AwHLHm/CcJtojQUFfAAzdOZnnLAX7y4lru/OtK/rFqFz+5cizlgUwnhmvoBzErPMUXXzwlLcXHiLJcRpTl8ulJETfr7oPHeWtDK2+s28Ora3bz7JJtZKT6OH9EKZdPqOTsYcWk+k28JhHEkwr5ODAfGC4iLSJyk4h8UUS+GD3keWATsAH4I/Bl20bbhdvOH0pNQRb/+eyKzttJuwmGFf4uX/zRlQF+PWs8y7e1cceflydkDIkm1LngT22Nj63K45GbJvP9T46msWk/H/vlG7z4nq3Xd4MNhE5xIe8NJbkZXD6hil/Nqqfxv2fy6OencNXEauZv2stNDzUy9Yev8L3nVrFpz2Grhm04BT2Ku1JqtlKqXCmVqpSqUkrdp5T6vVLq99HXlVLqK0qpwUqpMUqphLR6zEzz880LhrO59QgrtrUl4pSEwuEPidzHzijj9vOH8bdl23ln096EjCORxKy5nha8iHDN1FpeuO0sBhUP4EuPLubeNzclYogGiwhFbaQUvzVutVS/jxlDivjeJ0ez4D/P597rGpg8sICH5jVx3s9f59r7FvDyyp2Ew30Kvxl6wNP3R1MHFgCwZOv+hJwvGFLdWrBf+MggSnLS+cXL64hkhOpDp+Ue54KvK8rmiVumcsGoMr7/99V8Z+7KzvcwuJtTueCsINXvY+aoUu6+ZiLz/uM8vvHRYWzYfZhbHlnEzF++zpx3t3K8I2T5eZMZT4t7SW4GlXmZLGk+kJDzhcKqW5HLSPXz1fOG8G7TPt5Y35qQsSSKYBxumZPJSPVz12cncNOZA3lwXhO3P7HUCLwHiMcFZwUlORl89fyhvHnHufx2dj1ZaX6+/cwKzvrJazzw9mYj8hbhaXEHqK/JY+nWxIm7/xTBpk9PqqEyL5Ofv7xWK+v9fT9s774qfp/wPx8fxR0XDudvy7bzn8+s0OrvoiPxuuCsIsXv4xPjKvjbrWfy6OenMKgom//92yrO+em/eGR+Ex2hxMTSdEUDcc9n24Fj7Dp43PZzBcPdu2Ugkjlw28yhLG9p4x+rdtk+lkQRDPXPmvvyOUP46nlDeKKxme//fbUReBcT6mW2jFWICDOGFPHEF6bx2M1TqC7I5H/+ujIamN9pvjN9RANxzwMi1XV2E7HcTy1yl9dXMqgom1/8Qx/fe8gCa+7rHx3GDdPruO+tzfzu1Q1WDc1gMYm23Ltj+uAinvzCNO6/oYEUn/DFPy3i6j/M570EJU3ohOfF/YyKXNL8PpY02x9UDXaTLdOVFL+PL50zmDU7D7GwKTFBXrvpi8/9ZESEOz8+isvrK/n5P9aZNEmXEoq6Qez2ufeEiHDeiFJeuO0sfvipMWzac4RP/O4t/vsvKzhwtN3RsXkJz4t7eoqfURW5rrDcAS4ZW86A9BSeamw+7XFewQrLHcDnE354+RjGV+fx9SeXsWbnQSuGZ7CQTsvdolTI/pLi9/GZKTW8+s1zuH5aHY8t2Mq5P/sXzyxu0ebO2E48L+4Qcc0sbzlA0OYATMcpUiG7kpWWwiVjyvn7ih0cORG0dTyJoLdVi6cjI9XPH66dyID0FG5+uJH9R4wV5iYSlS3TWwKZqXzn0jP4+9fOYmBRNl9/chnX3f8uzfuOOj00V6OJuOdzvCPMmp2HbD3P6bJlunJVQxVH20M8v8L77ofObBmLrLnS3Ax+f+1EdrWd4GtzlpgCFhfhBp/76RhZnsvTX5zO9y47gyVbD/DRX77Og29vNt+hU6CHuFdHg6o257sHw+G4inkm1uYzqCibpxa12DqeRGCFz/1kJtTkc+cnRvHm+lbuf3uzZe9r6B9OZcv0Bp9PuHZaHf/4+tlMHVTId/62imvuW0DLfmPFn4x7Z7EXVOVnUjQg3fZK1Xh87hAJCF0xsYp3N+9jy94jto7JbkIhe6y5z06p4aOjSvnJi2tZtd34391A7ELuUsP9A5QHMnnghkn86PIxLGs+wIW/epNnl3jfmLISLcRdRBJSzHS6PPeTuWJCFT6Bpz1uvXfeqlu8EYOI8OMrxhLISuW2OUtMVaILiPVO8sqmGyLCrMk1vHj72Ywqz+XfnljG7XOWcOh4h9NDcwVaiDtEgqqbWo/YFqQLhxVKxW/BlgUyOGtoMX9e1OJpn2AorPAJtvRoL8hO4+dXjWP97sP88PnVlr+/oXcE47wzdRvVBVk8fstU/m3mMOYu284lv3mLZQlqSeJmtBH30RUBANuCqjELtje9qC+fUMn2tuMJ631jB5G7Ffu+JmcPK+bGGXU8PH8LCzTsquklQnFkg7kVv0+4beZQnvzCNEJhxVW/n88j85uSOmVSG3EfWBTZLWbrPnt83H3J9z5neAkpPuGfq73bjiAUDttuzX3rguFU5WfyH8+uMO4ZB/Gq5d6VhroCnvvqmcwYUsj//HUlX5uzlMMapCT3BW3EvTyQQapfaNprT9Q81g61N5ZNIDOVKYMK+KeHe830Js7QV7LSUjorEe96zbQncIpI11PvS0J+dhr3XT+Jb10wnL8v386n7nqbza3eTmzoC96fySgpfh/V+Vm2Zaf0tVJz5shS1u8+TJNHv1yhsEpIxeLZw4q5vL6Su/+10VSvOoQOlnsMn0/4yrlDeOSmKbQePsGlv3uL19budnpYCUUbcQeoLcyiqdUuy71v+d4zR5YCeNY1kwjLPcZ/f3wUuZmpfPvPKzwdhPYq3e005nVmDCli7q1nUpWfxeceXMjd/9qYNH54zcQ9my17j9gyee9b7r37k1UXZDGiLMez4h4KJc6aK8hO478uHsnS5gM8vdjbKaReRCfLvSvVBVk886XpXDKmnB+/uIZvPrWcE0H9YztaiXtdYRZH2kO0HrY+HbI/lZozR5aysGm/Jzva2Z0tczKfqq+kviaPn7y41uQrJ5hQAu/SEk1mmp/fzq7n9plD+fPiFq65dwF7D59weli2opW410YzZuzwu/enUnPmqFJCYcW/1u6xeli2k4hsma74fMJ3PnEGrYdP8FvT+z2h6Gq5xxARbp85jN/Mrmd5SxuX3z2PTXsOOz0s29BK3OsKI+JuR8ZMRyxbpg/BxbGVAYpz0vmHB10zifS5xxhXncdVE6t44O3NWi8+txHJc9dKErrl0nEVPH7LVA4dD3LF3fNYtGWf00OyBa1msjIvE79P7LHc+9Exz+cTZo4s4fW1e2gPemtfyHj76VjNty4cTnqKn+89tyrh505WdLfcuzKhJp9nvjSdQGYqs/+4QMsNZLQS97QUH5V5mbZY7v3dS/S8EaUcPhFk0RZv7dDk1IIvycnga+cP4bW1e3h7Q2vCz5+MhOLseqoLdUXZPPPlGYyuyOVLjy7m0QVbnB6SpWgl7hBJh7TXcu/bn2zqoAL8PmHeRm8JVaSwxZkFf920OirzMvnRC2tMamQCSCbLPUZBdhqPfn4q5w4v4b+efY9f/3O9NqmS2ol7XWG2LQVDfalQ7UpORipjKgPM2+it/inBODcosYOMVD9f/+gwVmxr43kNb5vdhs7ZMqcjMy2yQ9jlEyr55T/X8Z25K7UwJrQT99rCLA4eD1qedti5kUE/rNgZQwpZ1nzAU70unC5s+WR9JSPKcvjpS2vpsHkbxWQnGS33GKl+Hz+7chw3nzWQh+Zv4ZtPL7N920670U7c7cqYsWILshmDiwiGFe9u9o71HkxgEVN3+H3Cv184gi17jzLn3a2OjSMZCCW4psFt+HzCf148km98dBjPLN7GrY8t8XSxk3YzWVeUBVif627FFmQTavNJS/Exb4N3xN0Nt+rnDC9mysACfv3Kei02HXcryWy5xxARvnr+UO78+CheXLmTmx9e5NlOpdqJe1V+FiJY3mPGCss9I9VPQ20+b3vI7+6GBS8i3HHhCFoPt/PIO3plNLgJp11wbuJzZw7kJ1eM5c31e/jcgws52u49oyIucReRC0VkrYhsEJFvd/N6jYi8JiJLRGS5iFxs/VDjIyPVT0Ug0wbLvX8B1RjTBxeyesdBz5Q+u8Fyh8im42cPK+aeNzYZ690mnHbBuY2rJ1Xzi6vH8c6mvdzwwEJPxcogDnEXET9wF3ARMAqYLSKjTjrsv4EnlVL1wCzg/6weaG+oLcyiyWJxD1q0UfT0IUUAvLPJG1VxTmbLnMy/zRzKviPtPDS/yemhaImTaa9u5VP1Vfx6Vj2Ltuzn+vvf9ZTAx7NqJwMblFKblFLtwBzgspOOUUBu9OcAsN26IfaeSHdIa90yVmTLQKQVwYD0FN72SL67m27V62vyOWd4xHr30iLzCiEXXcjdxCfGVfC72fUsbT7ADR4S+HhmshJo7vK4JfpcV74DXCMiLcDzwFctGV0fqS3MYu+Rdg5a2FWwP10hu5Li9zFlYAHzPFJ1GUzQZh3xcvvMYRw42sFD85qcHop2ONFHyCtcNKac386uZ0nzAW584F1PuAatukzPBh5USlUBFwOPiMiH3ltEbhGRRhFp3LPHvg6JdYXRjBkLg6r9rVDtyvQhRTTtPcq2A8f6/V524xafe4zx1XmcN6KEe97YZFoCW4xTfYS8wsVjyvnNrHoWbz3AjR4IssajVNuA6i6Pq6LPdeUm4EkApdR8IAMoOvmNlFL3KKUalFINxcXFfRtxHFTlR8R92wHrxD1WQGOF0E0dVADAws3u97u7Mch22/lDaTvWwaMLTN67lQRd5IJzK5eMLedXnx5PY9M+bnF5mmQ84r4QGCoiA0UkjUjAdO5Jx2wFzgcQkZFExN2x5uUVeZkAbD9w3LL37E9XyJMZUZbLgPQUFja5X9zdZrlDpCXwWUOLuPfNza5eXF7DWO7x8YlxFfz0ynG8taGVLz+62LWdXnsUd6VUELgVeAlYTSQrZqWIfFdELo0e9g3gZhFZBjwO3KAc7L6Tn5VKeoqPnQetE3erfO4QuUDU1+R5okOkm7JluvLlc4bQevgETy0y2/FZhfG5x88VE6v4wadG8+qa3Xzt8SWubFUQ16pVSj2vlBqmlBqslPpB9Lk7lVJzoz+vUkrNUEqNU0qNV0q9bOege0JEKA9ksN1Cn7aVljvApLoC1u46RNsxd/uN3ZQt05WpgwqYUJPHH17faHrOWERkv1z3Xcjdymen1HZWst7x9HLXNRvTdibLAhnsbLPBcvdb8ydrqM1HKVi81d3WuxsqVLtDRPjKuUNo2X+Mvy1zNPNWG4Imz73XfO7MgZFeNEu28b9/W+mqdsHaintFIJMdFoq7VRWqMcbX5OH3CYua3C3ubvS5xzhvRAkjynL4v39tdJ3V5EWMz71v3HrekM5ukj9/eZ3Tw+lEW3EvC2Sw6+DxTndKf7Git0xXstJSOKMi1/VBVbfluXdFRPjSOYPZsPuwJ/endRsmW6ZviES6Sc6aVM3vXtvAvW9ucnpIgMbiXp6XSTCsaLWoh0uon9vsdcfE2nyWtRxwbbQd3G25A1wyppzKvEzXLCivEg4rwso64yXZEBF+8KkxXDymjO//fTV/dkGgX19xz80AsMw1Y7XlDpGg6vGOMCu3t1n2nlailHJ9SXqK38dNZw5kYdN+18cv3ExIWW+8JBt+n/DLT49n+uBC7vjzcl5d4+zdpHtXbT8pz4uKu0UZMzF/pIh1X/6G2nwA16ZEhixM/7STqydVk5uRYqz3fmBlBXYyk57i557rGhhVnsuX/rSYRVucc7tqO5PlgUghk5WWu9W3rCW5GdQUZLnW727H3YodDEhP4bNTa3nxvZ22bI6eDFhZx5HsDEhP4cEbJ1GRl8nnHmxkw+5DjoxDW3GPFTLtaLPKcrcn2NRQm8+iLftdlUIVwyuWO8AN0+vw+4T739rs9FA8SSym5PPAXHuBwgHpPPy5yaSl+Lj+/oWWpmXHi7biHitkcrPlDjCxLp/Ww+2Wtyi2Aq9Y7gCluRlcNr6SJxtb2H/E2s3Rk4Ggxam+BqguyOKBGybRdqyD6+9/N+EFi9qKO0RcM1aJu11ZIw21kSZibvS7e8lyB7j5rEEc6wjxmNlIu9dYXYFtiDC6MsDvr5nIptbDfPGRRQndcFtzcbeuSrXDptLsISUDyE7zs7T5gOXv3V9i1pzfoqpcuxlelsOZQ4p4ZP4W05Kglxifu32cObSIn1w5lvmb9vLvTy9PmAvWG6u2j5TnZbDTokImu3zufp8wtirPleLuNcsd4MYZdew8eJwX3tvp9FA8hbHc7eVT9VV864Lh/GXpdn728tqEnFNrcS8LZBKyqJDJzh4r42vyWL3joOva11q1b2wiOXd4CXWFWTzwtgms9ob3eyd5Z669xpfPGczsyTXc9dpGnlzY3PMv9BOtxb0iEMl1t6I7ZCisSLXpiz++Oo9gWLmumMmLlrvPJ9wwvY4lWw+wxBQ1xY3Jc7cfEeF7l53BddNqmRLdsMdOtJ7Jsqi4W+F3t9Nyr6/OA2DJVne5ZryULdOVKxuqyUlP4YG3m5weimfw4oXci6T4fXz3stHUFmbbfi6txb0iWsi03QJxD4UUKTZZNSW5GVQEMlznd39/wXvrazIgPYWrJ1Xz/IodjuQXe5HO4LkRd23w1qrtJXmxHZksKGSyu6/5+Jo8lrW4S9y9vOBvmF5HSCkeeafJ6aF4AmO564fW4i4iVORlWmO5h8O2BpvGVeXRvO8Yey3qYmkFXl7w1QVZnD+ihDnvNic0t9ireNUFZzg1Wos7QFmuNbnutlvuUb+7m1wznQveoxkU10+vY++Rdl5YYdIie8KrLjjDqdF+JsvzMizpDGl3X/MxVQH8PnGVuHvZcgeYMbiIQcXZPDS/yemhuB4vpr0aTo/+4h7IYNehE/0uZLLbcs9KS2FYaY6rxN3rC97nE66bWsuSrQdY7rJ4htsImTx37UgCcY8UMu051D9fdsRyt/fPNb46Uqnqlv1AdbhVv2JiFdlpfh6ev8XpobgaLwfPDd3j3VUbJ+WB2I5M/XPN2G25QyTf/dDxIJta3dGTXIcFn5ORyuUTqpi7bDv7TLfIU+J1F5zhwySBuFuzaYddvWW6Mr7GXUFVXRb8ddNqaQ+GeSIBJd9exWTL6If24l6amw7AroP9E/dgyH7LfXDxALLS/Ly3zR1tCHRZ8ENLc5g6qIDH3t1iSRM5HdHBBWf4INrPZH5WGql+YXc/fe7BsLI92OT3CaMrAq4J/ukUZLtmai3N+47xxro9Tg/FlehyITe8j/bi7vMJJTkZ/bbcExFQhUhK5MrtBwm6oB+5Tj2+PzaqjKIB6fzpHRNY7Y6Q2YlJO7QXd4CS3HR2H+yv5W6/zx1gbFWAE8Ew63Ydtv1cPRHqDKh6/2uSluJj1qRqXl27m+Z97tvS0Gm8nvZq+DDeX7VxUJKT3n/LPQE+d4CxVZGg6optzrtmYgteF2tu9pQaBHjcbMP3IXRywRkiJIW4l+ZmeMLnDlBbkEVORgrLW5wPquq2O09lXibnjSjlycZm2oPOu73chPG560fSiHvbsY5+7XQUSkCeO0RiBGMqA6xwQcaMTj73GNdMraH1cDsvrjT9ZrpismX0I66ZFJELRWStiGwQkW+f4pirRWSViKwUkcesHWb/KM6JpEP2x+8eTFBAFSJB1dU7DjrezVA3yx3g7KHF1BRk8agJrH4AY7nrR49qJSJ+4C7gImAUMFtERp10zFDgP4AZSqkzgNttGGufKc2NVKnuOtR3v3uiLHeAsZV5dIQU63Y6G1QNamjN+XzCrMnVLNi8jw27nQ9auwWTLaMf8azaycAGpdQmpVQ7MAe47KRjbgbuUkrtB1BK7bZ2mP0jVsjUP8s9MdkyEMmYAVjucFC1M1tGsyDbVROrSfGJCax2wVju+hGPuFcCXeu2W6LPdWUYMExE3haRd0Tkwu7eSERuEZFGEWncsydxxSSlOVHLvR8ZM4m03KvyM8nLSmWFw0FVHX3uEHHTXTC6jKcXtfQrDqMTIc0yowzWBVRTgKHAOcBs4I8iknfyQUqpe5RSDUqphuLiYotO3TN5Wamk+X39cssEbe7n3hWRSFDV6YyZkMa5z5+dXEPbsQ6eX7HD6aG4AmO560c84r4NqO7yuCr6XFdagLlKqQ6l1GZgHRGxdwUiQnFOOnv66JYJhxVKJbaYZ1xVHut2HXLUsuxc8KLfgp82uJCBRdk8tsC4ZuD9O1PRcK6TlXjUaiEwVEQGikgaMAuYe9IxfyFitSMiRUTcNJssHGe/Kc1N77PlHnSgwGNMVYBgWLF6x8GEnfNkQmGFTyJBSN0QEWZPrqZxy37W7Trk9HAcJxEtrQ2JpUdxV0oFgVuBl4DVwJNKqZUi8l0RuTR62EvAXhFZBbwGfEsptdeuQfeF0twMdvXRcneir3lnUNVB10wi0z+d4MqJ1aT5fcZ6JzEtrQ2JJa6Vq5R6Xik1TCk1WCn1g+hzdyql5kZ/VkqpryulRimlxiil5tg56L7QnxYETgQWy3IzKBqQ5mgxUygc1tqaK8hO46IxZTyzuIVj7ckdWDWWu37oa5adREluBoeOB/u0iJ3IJBARRlcGHO3tnsggslPMnlzDwePBpA+s2r0BvCHxJI24xwqZdvfB794ZWPQn9s81pjLA+t2HHQuqhsJKuxz3k5kysIBBRdlJn/MesdyTRg6SgqSZzfd3ZOq9392p7ebOqAgQcjComgyWeySwWpP0gdVQSP+5TjaSRtxL+lHI5NRG0WOiQVWnXDOJanPsNFdMrCLN70tq69343PUjacS9swVBH1r/OmW5VwQyKMh2Lqiqe7ZMjILsNC4YXcYzi7clbcVqKBw2vdw1Q/+VGyWQmUpaio/dfbLcnaneiwVVV2xzxi2je7ZMV2ZPrqbtWAcvvJecgVVjuetH0oi7iPQ5HdLJXtdjKnNZ71ClajL43GNMG5TcFasmW0Y/kkbcoe87Mjm5v+Toikil6tqdiQ/2JbJZmtOICJ+eVM3Cpv1s2J18gVWTLaMfSTWbpbn9tdwdEPfKSFDVCb97st2qXzGhihSfMOfd5p4P1gxjuetHUol7SU5Gn3q6Bx3sax5r/+tExkwoQfvGuoXinHQ+dkYpf17c4vguWIkm2S7kyUByiXtuOodOBDnaHuzV7zlpucfa/zpnuSfVV4RZk2rYf7SDl1bucnooCcX0ltGPpFq5sU07emu9O93renRlgHW7DiXcmkzGBX/mkCKq8jOZk2Q578EkqWlIJpJL3HP7VsgUC6imJrj9QIwxlQFH9lRNxgXv8wmzJlUzb+NemlqPOD2chJFsLrhkIMnEPdqCoJcZM05VqMYYXeFMUDVZg2xXNVTj9wlzFiZPYDUZXXC6k1SzWdLplumd5e6kzx2guiCTQGZqwsU9WYNspbkZnDu8hKcXtdARCjs9nISQrBdynUkqcc/NTCE9xdfrXHenfe6RStXchGfMJPOC/8yUaloPn+CV1ckRWE3WC7nOJJW4i0h0R6a+Wu7O/blGVwZYu/MQ7cHEWZLJfKv+kWEllAcyeCxJct6TMXiuO0m3cvvSgsBpyx0iQdX2UDihbWmTecH7fcJVDdW8uX4PzfuOOj0c2zGWu34knbj3pQVBKBpQdVLoYkHVRLpmgkmwWcfp+PSkagCeatTfek9mF5yuJJ24l+Sm9z7P3cHeMjGbAwobAAAZ1klEQVRqC7PIyUhJaFA12Rd8ZV4mHxlWzBONzQQ1D6xG0l6TTg60JulmszQ3g8Mnghw+EX+VaqfP3UErVkQYXZHYPVWTMc/9ZGZNqmHXwRP8a+0ep4diK5EmcU6PwmAlSTedJTnRTTt64Xd3g88dIjszrd55KGHpecluuQOcP7KEogHp2u/SlMzBc11Jutl8f6Ps+F0zbsiWgUjGTHswcUFVs+AjVclXN1Tx2trd7Gg75vRwbCOZg+e6knQr9/2Nsj1ouUfb/65M0M5MZsFHmDWphrCCJxe2OD0U2zDZMvqRdOJektv75mFuyJYBqC3IYkB64oKqZsFHqCnM4qyhRTyxcGvnXZxuGBecfiSduOekp5CR6uuT5e50YyWfTzijIjdh4m4W/PvMnlzD9rbjvLFez8BqKMnTXnUk6cQ9VqXaG597LBXSaZ87RFwzq3ccTEhqXrLnuXdl5shSigak8bime6yaC7l+OK9WDlCa07sWBDHL3Q3f/TFVAU4Ew6zfbX/7X7Pg3yctxceVE6t5Zc3uPm3V6GaUUiZ4riFJOZvFuem9zJaJBBZFnBe6RO2pqpSK5j4n5VekW2ZNqiYUVtpVrMbCCOZCrhdJuXJLczJ6nefulsDiwMJsstP8thczOd3m2I3UFWUzY0ghj7/brFVg1en9Cgz2EJe4i8iFIrJWRDaIyLdPc9wVIqJEpMG6IVpPaW46R9pDcVephkLucU/4fMIZCdhT1S3pn25j9uQath04plVg1VzI9aRHcRcRP3AXcBEwCpgtIqO6OS4HuA1YYPUgraa32+25yXIHGFcVYNX2g7ZWqpoF3z0fG1VG0YA0HtMosGou5HoSj+U+GdiglNqklGoH5gCXdXPc94AfA66PNsVaEMQr7pH9Jd3jwRpTlRcJqu6yL6hqFnz3xAKrr67Zzc4213/V4yIUMhdyHYlHsSqBrhGkluhznYjIBKBaKfV3C8dmG7FCpj1xBlXdZrmP7QyqHrDtHMZyPzWfmVxDKKx4QpM9Vjsv5C4yYAz9p9+zKSI+4BfAN+I49hYRaRSRxj17nPNZ9rYFgdvK8GPtf5e12Od37wyymQX/IWIVq3MWbtWiFbC5kOtJPCt3G1Dd5XFV9LkYOcBo4F8i0gRMBeZ2F1RVSt2jlGpQSjUUFxf3fdT9ZEB6CpmpfnbF2YLAbZa7iDC2KsAKG8XdLPjT89kpNexoO65FK2CTLaMn8Yj7QmCoiAwUkTRgFjA39qJSqk0pVaSUqlNK1QHvAJcqpRptGbEFRKpU499uz43FPGMq81iz8yAngiFb3t8NG5S4mfNHllKck85jGrQCNhdyPelR3JVSQeBW4CVgNfCkUmqliHxXRC61e4B2UdKLFgRus9wBxlYF6Agp1u60p/2vWfCnJ9XvY9akal5bu9vze6ya4LmexOVQVUo9r5QappQarJT6QfS5O5VSc7s59hw3W+0xSnPjL2QKhRSpLvM9x9r/LrfJNWMWfM/MmlyDgOc38nDLfgUGa0na2SzJSWfXwRMo1XOloRst96r8TPKzUm3zu5sF3zOVeZmcN6KUJxubaQ96N7BqXHB6krQrtzQ3nWMd8VWpBl2WLQORuMGYqjyW21SpaoJs8XHN1BpaD7fz4sqdTg+lzxgXnJ4ksbjHqlR79ruHXGi5QyTffd2uQxzvsD6oahZ8fJw9tJiagiz+9M4Wp4fSZ95PezVzrRNJK+4lObEdmXr2uwdDypXuiTFVAUJhxcrt1m+7935hi1nwp8PnEz4zpYZ3N++zLbhtN+ZCrifuU6wEURIrZDrUs7i71XIfV5UHwIoW6ytVzYKPn6sbqklL8fHoAm9a7yZ4ridJK+69ccsEw2HHt9jrjtLcdIpz0m3JmDFBtvgpyE7jkjHlPLN4W9ydRt2ECZ7rSdLO5oD0FAakp8TV/MmtlruIMK4qwFJbLfek/Yr0imun1XL4RJBnl2zr+WCXYSx3PUnqlVseyGBH27Eejwu6sEI1xvjqPDbtOULbsQ5L39dky/SO+uo8xlQGeHheU1zptW4iFJ1rt37HDX0jqcW9LJDhacsdYHx1PgDLLbbejc+9d4gI106rZf3uw7yzaZ/Tw+kVxgWnJ0kt7hWBTLbHIe4Ry92df6oxVZFK1aVbrRV3c6veey4dV0FeVioPz29yeii9ovNC7sK4kqHvuFOxEkRZIIPWwyd6rC50s+UeyExlcHE2S5ttstzNgo+bjFQ/n55UzcurdrH9QM/uPrcQNHdpWpLU4l6Rl4FSPfd1d2OFalfGV+eztPmApb5es+D7xjVTagkr5alt+EKdd2lJLQfakdSzWR7IBGBHD66ZUEi52oIdX5PH3iPttOy3zloMdQZUk/or0muqC7I4f0Qpj7+71bZ2zFZjLuR6ktQrtzwQyXXvKWMm0jjMvX+q+upIMZOVrpmg2Vezz1w/vZa9R9r527IdTg8lLkImM0pL3KtYCaA8L07L3cWpkADDy3JIT/FZKu4hE1DtM2cOKWJoyQAeeHuzJ9IijeWuJ0kt7gPSU8iJo5CpIxR2tcil+n2MqQxYa7mbBd9nRIQbZwxk5faDvLvZ/WmR5kKuJ0kt7gDleRk9Zja43XKHSDHTe9va6LBow2az4PvHp+oryctK5YG3m5weSo+874JLejnQiqSfzbJAJjt7zJZRru+OOK46jxPBsGWdCYOm/UC/yEzzM3tyDS+v2un6bfg6L+Qu/44bekfSr9yKQAbbD3jb5w4Ryx1giUWumZDp8d1vrptWi4i4vqjJuOD0JOnFvadCJqWU67NlILLtXtGANMsqVc2C7z/lgUwuGl3GnIXNru4WabJl9MTdipUAKqK57qcqZIpqnOtFTkQYX53Pkq37LXm/kOk3YgmfO3Mgh44Heaqx2emhnJLOVhNi5lonkl7cyzpz3bsXdy91R2yoy2dT6xH2Hu65R31PmAVvDRNq8mmozee+tzYTtCjYbTWhsMInkV2lDPqQ9OJekXf6QiYvdUecVBfpENm4pf/Wu1nw1nHz2YNo2X/MtZtou7kxnqHvJP2MlvXQgsBL3RFHVwZIS/GxyAJxNwveOmaOLGVgUTZ/fGOTK4ua3NwYz9B3kn71DkhPIScjhR2nyHUPeagMPz3Fz7iqAAub+l84Ewq7u3DLS/h9wk1nDmRZS5sri5oiG8CbudaNpBd3OH1f986sEb83/lQTawt4b1sbxzv617TKzbtPeZErJlRRkJ3GPW9scnooHyIUDpuUVw3xhmLZzOl2ZPKSzx2goTafjpDq96bZIQ8UbnmJzDQ/106t5ZU1u1m/y5pCM6swF3I9MeJOJKh6qoCql7JlACbWRoKq/XXNmAVvPddNqyUj1cfvX3eX9W587npixB0oy82k9XB7t/23vbYjUX52GkNKBvQ7qBoKmQVvNYUD0pk1qYa/Lt1Gy373tCQwwXM9MTNKpHkYwK62D+eHd4S8t0vNpLp8Gpv2EQ73PTPDLHh7uOXsQQD80UW+d2O564lZvZx+0w6v+dwhElQ9eDzIhj2H+/weJlvGHiryMrl8QiVzFjaz51D/i82swLjg9CQucReRC0VkrYhsEJFvd/P610VklYgsF5FXRKTW+qHax+m22/Oazx3eL2bqj9/dLHj7+OJHBtMeCnP/25udHgpgLuS60qO4i4gfuAu4CBgFzBaRUScdtgRoUEqNBZ4GfmL1QO2k/DQtCLxoudcUZFE0IJ1FTX33u5tbdfsYVDyAi8eU86f5W2g71uH0cAia+IqWxGO5TwY2KKU2KaXagTnAZV0PUEq9ppSKRYjeAaqsHaa9ZKenEMhMZduBDwe5vFShGkNEmFSXz4J+FMwEjbjbypfPGcyhE0Eemtfk9FAiLa09kjBgiJ94xL0S6NrSriX63Km4CXihP4NygtrCLLbs/bC4v2+5eys8MW1wIdsOHGNrN58pHsyCt5czKgLMHFnKvW9uctx690JLa0PvsXRGReQaoAH46Slev0VEGkWkcc+ePVaeut/UFmZ3K+5Bj7a+nT64CIC3N7b26ffNgref22cO5eDxIA847HsPhRXmOq4f8azebUB1l8dV0ec+gIjMBP4LuFQp1W0agFLqHqVUg1Kqobi4uC/jtY26wixa9h/90KYdXstzjzG4OJuSnHTmbdzbp98PhcOeijN4kdGVAT42qpT73trsqPUeDIc9d2dq6Jl4ZnQhMFREBopIGjALmNv1ABGpB/5ARNh3Wz9M+6ktzCasYNtJDcS8mC0DEb/7jCFFzN/Y2qdOhCbIlhhunzmMQ8eD3PeWc9a7CZ7rSY/irpQKArcCLwGrgSeVUitF5Lsicmn0sJ8CA4CnRGSpiMw9xdu5lrrCLACa9h75wPMxyz3Vg5bN9MGFtB5uZ20fepl4Yd9YHRhVkcuFZ5TxwFubaTvqjPUeNPEVLYlLsZRSzyulhimlBiulfhB97k6l1NzozzOVUqVKqfHRf5ee/h3dR21hNgBbWj8o7l7MlokxfUjE7z5vQ+9dMyZbJnHcNnMoh04E+cMbGx05f9jMtZZ4zxy1iaIBaWSl+Wk6KajqVZ87QGVeJnWFWczrQ1DVWO6JY2R5LpeNr+D+tzefci9fOzEFa3pixD2KiFBbmM3WfR8Udy9b7gDTBhexYNO+Xu/fabJlEss3PjqcUFjxq3+uT/i5jc9dT8zq7UJdYVY3PveIKHrVspkxpJBDJ4Is39a7/u4mWyax1BRm8dkptTzZ2MyG3X3vCdQXTJM4PTEz2oXawmya9x3tdMVA166Q3hS6aYMKAZjfy5TIoNmsI+F89bwhZKb6+dlLaxN6XmO564kR9y7UFWbREVJs75IO6dUK1RiFA9IZUZbD2xt653c3PvfEUzggnVvOHsSLK3eyeGv/NzmPl6C5S9MSbyqWTXRmzHQJqnrd5w5w1tAiGpv2c/hEMO7fMXnuznDTmQMpzknne8+t6lc//t5gNmbREyPuXagr+nCueyjkbZ87wPkjS2kPhXlzXfwtH4zl7gzZ6SncccFwlmw9wF+WfqgQ3BZMnrueGHHvQmlOBukpPrZ0EfdOy93DX/6G2nwCman8Y/WuuH/HZMs4xxUTqhhXncePXljTq7utvmJ87npiVm8XfD6htjDrA7nuXuznfjIpfh/njSjhtTW7406JNNkyzuHzCd/5xCh2HzrBXa9tsP18JltGT8yMnkSkO2Q3lrvHhW7myFL2H+1g8dYDcR1vKlSdpb4mnysmVHHfm5tpOqlq2mqM5a4nRtxPoi7a1z0WzPJ6tkyMs4cVkeoX/hmna8b43J3n3y8cTlqKjzvnruxT87d4MdkyeuJtxbKB2sJsTgTD7DoUKQMPhhUi3rfcczJSmTqokH+uik/cTZ6785TkZvDNjw3jjXV7+OvS7badx1juemLE/STqoumQTa0Rv7tOvuePjiplU+sRNu7puQLSWO7u4NppddTX5PHd51ax70i7LecwvWX0xIj7SdRGW//G/O46+Z7PH1kK0KP1rpSKWnPm6+E0fp/wo8vHcuh4B99/bpXl7x8OK5TCzLWGmBk9ifJABql+6cyYCYX0ySSozMtkVHluj353HTKEdGJ4WQ5f/MhgnlmyjTd6UasQD7GEAZPnrh96qJaFpPh91BZmsz66wYVOljtEXDONW/azo+3YKY/RJUNIJ75y7hAGF2dzx9PLOXDUOvdMyMy1thhx74ZxVXksbT7Q6Z7QyYK9fEIlSsEzi09d/Wgsd/eRkernV5+up/XwCf7z2RWWZc8EPd711HBqjLh3Q31NHnuPtNO87xjBcFgrq6a2MJvJAwt4qrH5lAJhLHd3MqYqwDc+NpznV+zkqUUtlrynsdz1xYh7N9TX5AGwpHk/wZBeljvAVROraNp7lMYt3XceNJa7e/nC2YOYNqiQ78xdyWYLipuCZq61xYh7NwwvzSEz1c+SrQciWSOaBZsuHlNOdpqfpxqbu309dqvu95uvh9vw+YSfXz2OVL+PWx9bzLH2UL/e733L3cy1bpgZ7YYUv4+xVQGWbN2vZd+N7PQULhlbzt+X7+Bo+4cbUxnL3d1U5GXyi6vHsWrHQb79zPJ++d+N5a4veqmWhdTX5LNy+0GOnAhq6Y+8qqGaI+0hnl+x80OvBT2++1QycP7IUr7x0WH8del2/vjmpj6/T8jMtbYYcT8F9TV5BMOKZS1tWlo1DbX5DCzK7tY1Yyx3b/CVc4dwyZhyfvTCGl7vY/57Z7aMZq5HgxH3U1JfHQmqth4+oaVVIyJcObGKBZv3sXL7BzfPNtky3kBE+OlVYxlWmsNXHl3Msub4On52xWTL6IsR91NQkptBZV4mEPHB68g1U2vJzUjhFy+v+8DzunTCTAay0lJ48MbJ5Gencv0D77J256Fe/b7xueuLWb2nIZYSqesXP5CZyhc+MphX1uz+wIbMndkymn5u3SgLZPDoTVNJ8/u45r4Fver/brJl9MXM6Gmor8kH9Ba5G6bXUZid9gHr3fjcvUdNYRaPfn4KwVCY2X98hzU7D8b1e8Zy1xcj7qdBd8sdImmRXzpnMG9taGX+xr2AHvvGJiNDS3N49PNTCYUVV909n7c3tPb4OyFzl6YtRtxPwxkVuaT5fdp/8a+ZWktpbjo/e3ltZz8d0PuipiujKnJ59iszKM/L4Pr73z1loVqMWNqrmWv9MOJ+GtJT/EwbXEhFINPpodhKRqqf284fxqIt+/n965tMnrvHqczL5KkvTmfywAK+9fRybpuzhLajHd0ea7Jl9CXF6QG4nT9e15AUX/zZk6uZt7GVH7+4hmun1gImW8bLBDJTeehzk/m/1zby21fX886mvfzoirGcO7zkA8eZfu76EtfqFZELRWStiGwQkW9383q6iDwRfX2BiNRZPVCnSEvR3y0DkZzpn101jvqaPB55ZwtgrDmvk+r3cdvMoTz75RnkZKRy4wML+fQf5jNvY2tnywKTLaMvPc6oiPiBu4CLgFHAbBEZddJhNwH7lVJDgF8CP7Z6oAb7yUj1c8+1De/n9xtx14IxVQGe++qZ3PnxUWxuPcJn/riAy++ex71vbmJTNG3SzLV+xOOWmQxsUEptAhCROcBlQNcNHS8DvhP9+WngdyIiyqodBQwJozgnnQdunMTvXt3A4JIBTg/HYBEZqX4+d+ZAPjOlhicbm3lk/ha+//fVna/7xIi7bsQj7pVA15B7CzDlVMcopYIi0gYUAh/IxRKRW4BbAGpqavo4ZIPdDCvN4Tez650ehsEGMlL9XDetjuum1dG87yivrtlN094jDC01F3LdSGhAVSl1D3APQENDg7HqDQYHqS7I4vrpdU4Pw2AT8URRtgHVXR5XRZ/r9hgRSQECwF4rBmgwGAyG3hOPuC8EhorIQBFJA2YBc086Zi5wffTnK4FXjb/dYDAYnKNHt0zUh34r8BLgB+5XSq0Uke8CjUqpucB9wCMisgHYR+QCYDAYDAaHiMvnrpR6Hnj+pOfu7PLzceAqa4dmMBgMhr5iKhcMBoNBQ4y4GwwGg4YYcTcYDAYNMeJuMBgMGiJOZSyKyB5gSx9/vYiTql+ThGT83Mn4mSE5P3cyfmbo/eeuVUoV93SQY+LeH0SkUSnV4PQ4Ek0yfu5k/MyQnJ87GT8z2Pe5jVvGYDAYNMSIu8FgMGiIV8X9HqcH4BDJ+LmT8TNDcn7uZPzMYNPn9qTP3WAwGAynx6uWu8FgMBhOg+fEvaf9XHVARKpF5DURWSUiK0XktujzBSLyDxFZH/0/3+mxWo2I+EVkiYg8F308MLov74boPr1pTo/RakQkT0SeFpE1IrJaRKYlyVz/W/T7/Z6IPC4iGbrNt4jcLyK7ReS9Ls91O7cS4TfRz75cRCb059yeEvc493PVgSDwDaXUKGAq8JXo5/w28IpSaijwSvSxbtwGrO7y+MfAL6P78+4nsl+vbvwaeFEpNQIYR+Tzaz3XIlIJfA1oUEqNJtJxdhb6zfeDwIUnPXequb0IGBr9dwtwd39O7Clxp8t+rkqpdiC2n6tWKKV2KKUWR38+RGSxVxL5rA9FD3sI+KQzI7QHEakCLgHujT4W4Dwi+/KCnp85AJxNpG02Sql2pdQBNJ/rKClAZnSDnyxgB5rNt1LqDSJt0Ltyqrm9DHhYRXgHyBOR8r6e22vi3t1+rpUOjSUhiEgdUA8sAEqVUjuiL+0ESh0all38CrgDCEcfFwIHlFLB6GMd53sgsAd4IOqOuldEstF8rpVS24CfAVuJiHobsAj95xtOPbeW6pvXxD2pEJEBwJ+B25VSB7u+Ft3pSptUJxH5OLBbKbXI6bEkmBRgAnC3UqoeOMJJLhjd5hog6me+jMjFrQLI5sPuC+2xc269Ju7x7OeqBSKSSkTYH1VKPRN9elfsNi36/26nxmcDM4BLRaSJiLvtPCK+6LzobTvoOd8tQItSakH08dNExF7nuQaYCWxWSu1RSnUAzxD5Dug+33DqubVU37wm7vHs5+p5or7m+4DVSqlfdHmp61611wN/TfTY7EIp9R9KqSqlVB2ReX1VKfVZ4DUi+/KCZp8ZQCm1E2gWkeHRp84HVqHxXEfZCkwVkazo9z32ubWe7yinmtu5wHXRrJmpQFsX903vUUp56h9wMbAO2Aj8l9PjsekznknkVm05sDT672IiPuhXgPXAP4ECp8dq0+c/B3gu+vMg4F1gA/AUkO70+Gz4vOOBxuh8/wXIT4a5Bv4XWAO8BzwCpOs238DjRGIKHUTu0m461dwCQiQbcCOwgkgmUZ/PbSpUDQaDQUO85pYxGAwGQxwYcTcYDAYNMeJuMBgMGmLE3WAwGDTEiLvBYDBoiBF3g8Fg0BAj7gaDwaAhRtwNBoNBQ/4/Ul+B4/42ZU8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=10,T_mult=2,eta_min=0)\n",
    "\n",
    "plt.figure()\n",
    "x = []\n",
    "y = []\n",
    "for epoch in range(100):\n",
    "    scheduler.step()\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "plt.plot(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. 自适应调整学习率 ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reduce learning rate when validation accuarcy plateau.\\\n",
    "# 当某指标不再变化（下降或升高），调整学习率，这是非常实用的学习率调整策略。\n",
    "# 例如，当验证集的 loss 不再下降时，进行学习率调整；或者监测验证集的 accuracy，当accuracy 不再上升时，则调整学习率。\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=5, verbose=True)\n",
    "\n",
    "\n",
    "for t in range(0, 80):\n",
    "    train(...);\n",
    "    val(...)\n",
    "    scheduler.step(val_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Lambda 算子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1193f7e50>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8HOW97/HPb4u06nKR5CLbcreFcQFhDDgUB7imOjmEgEkhB4JDEuckOcm54dzUyzn3nLRDSKGEQAoJoYVmCMahl4DBsnGTe7dcZVvFVl/puX/s2hHGRsJeabSz3/frta/dmXmk+c1r7K9mn5l5xpxziIiIvwS8LkBERBJP4S4i4kMKdxERH1K4i4j4kMJdRMSHFO4iIj6kcBcR8SGFu4iIDyncRUR8KOTVivv37+9KSkq8Wr2ISFJavHjxPudcQWftPAv3kpISysvLvVq9iEhSMrOtXWmnbhkRER9SuIuI+JDCXUTEhxTuIiI+pHAXEfGhTsPdzH5rZnvNbOVxlpuZ/cLMNpjZcjM7LfFliojIh9GVI/ffAzM/YPklwOj4aw5w18mXJSIiJ6PTcHfOvQYc+IAms4D7XcxCIN/MBiaqwKMt2VbND+evQY8HFBE5vkT0uQ8GtneYrozPex8zm2Nm5WZWXlVVdUIrW7mjlrtf3cjGqkMn9PMiIqmgR0+oOufucc6VOefKCgo6vXv2mC4cXwTA31btSWRpIiK+kohw3wEM6TBdHJ/XLQblZ3Dq4DyeV7iLiBxXIsJ9HvDZ+FUz04Ba59yuBPze47qotIil22vYe7CpO1cjIpK0unIp5IPAW8BYM6s0sxvN7GYzuzne5FlgE7AB+A3wpW6rNu6i0iKcgxdX7+3uVYmIJKVOR4V0zs3uZLkDvpywirpg3IAcivtk8PyqPcyeOrQnVy0ikhSS8g5VM+Oi0iLe2LCP+uao1+WIiPQ6SRnuEOuaaYm28/r6E7ukUkTEz5I23KeW9CUvI6xLIkVEjiFpwz0UDDBjXCEvrdlLtK3d63JERHqVpA13gItLi6hpaOWdzR80OoKISOpJ6nA/b2wBkXCA+St3e12KiEivktThnpkW4oKxhTxXsZv2dg0kJiJyWFKHO8DMCQOoOtjM4m3VXpciItJrJH24zxhXSFowwPwV6poRETks6cM9JxLm3DH9WVCxW2O8i4jEJX24A8ycMJAdNY0sr6z1uhQRkV7BF+F+0fgiQgHj2ZXdOhiliEjS8EW452WGOXtUf55bqa4ZERHwSbgDXDJhAFv3N7BqV53XpYiIeM434f6/ThlAMGA8s1xdMyIivgn3vllpTB/Vn6eX7VTXjIikPN+EO8DlEwdSWd3I0u01XpciIuIpX4X7xacMIC0YUNeMiKQ8X4V7XkaY88YW8MzynRprRkRSmq/CHWJdM3vqmlm0RcMAi0jq8l24Xzi+iEhYXTMiktp8F+5Z6SE+Or6IZ1fs0hOaRCRl+S7cAa6YOIj99S28uXG/16WIiHjCl+F+/tgCciIhnnx3h9eliIh4wpfhHgkHuezUgTxXsZuGlqjX5YiI9DhfhjvAx6cMpqGljedX7fG6FBGRHufbcD+jpC+D8zN4Ql0zIpKCfBvugYAxa/IgXl+/j6qDzV6XIyLSo3wb7hDrmmlrdzy9bKfXpYiI9Chfh/voohwmDM7lyaXqmhGR1NKlcDezmWa21sw2mNktx1g+1MxeNrN3zWy5mV2a+FJPzMcmD2Z5ZS0b9h7yuhQRkR7TabibWRC4A7gEKAVmm1npUc2+AzzinJsCXAvcmehCT9SVkwcRDBiPLan0uhQRkR7TlSP3qcAG59wm51wL8BAw66g2DsiNf84Dek0nd2FOhPPHFPD4kkoNRyAiKaMr4T4Y2N5hujI+r6MfAJ82s0rgWeArCakuQa4uK2ZPXTOvr9/ndSkiIj0iUSdUZwO/d84VA5cCfzSz9/1uM5tjZuVmVl5VVZWgVXduxrgi+mal8eji7Z03FhHxga6E+w5gSIfp4vi8jm4EHgFwzr0FRID+R/8i59w9zrky51xZQUHBiVV8AtJCAWZNHsQLq/ZSXd/SY+sVEfFKV8J9ETDazIabWRqxE6bzjmqzDfgogJmNJxbuPXdo3gVXnz6ElrZ2ntJlkSKSAjoNd+dcFJgLLABWE7sqpsLMbjWzK+PNvgHcZGbLgAeBzznnetVz7koH5TJhcC6PLtZVMyLif6GuNHLOPUvsRGnHed/r8HkVcE5iS0u8q08fwvfnVVCxs5ZTBuV5XY6ISLfx9R2qR5s1eRBpoQCPLNKJVRHxt5QK9/zMNC6ZMIDH391BY0ub1+WIiHSblAp3gOumDuVgU5Rnlvea+6xERBIu5cJ96vC+jCjI4sF3tnldiohIt0m5cDczrps6lCXbalizu87rckREukXKhTvAVacVkxYM8NA7OrEqIv6UkuHeJyuNS04dwGNLKnViVUR8KSXDHWC2TqyKiI+lbLifObwvowqz+dPCrV6XIiKScCkb7mbGZ88axrLKWpZur/G6HBGRhErZcIfYA7Sz0oLc/9YWr0sREUmolA73nEiYq04v5pllu9h/qNnrckREEialwx3gM9OG0dLWzkMab0ZEfCTlw310UQ5nj+zHAwu36hmrIuIbKR/uAJ89q4SdtU28sHqv16WIiCSEwh24cHwhg/Ii/P7NzV6XIiKSEAp3IBQMcP3ZJSzcdICVO2q9LkdE5KQp3OOunTqUzLQgv31DR+8ikvwU7nF5GWE+WTaEp5fvZG9dk9fliIicFIV7B/98TgnRdsf9b2lIAhFJbgr3Dob1y+Ki8UU88PZWjRYpIklN4X6UG6cPp7qhlcffrfS6FBGRE6ZwP8rU4X2ZWJzHfa9vpq3deV2OiMgJUbgfxcz4wrkj2bSvnr9V7Pa6HBGRE6JwP4aZEwZQ0i+Tu1/diHM6eheR5KNwP4ZgwLjp3BEsq6zlrY37vS5HRORDU7gfx1WnFdM/O527Xt3odSkiIh+awv04IuEgN0wv4fX1+zQkgYgkHYX7B/jUmcPITg/p6F1Eko7C/QPkZYT5zFnDeHbFLjbsPeR1OSIiXaZw78Tnpw8nEgpyx8sbvC5FRKTLuhTuZjbTzNaa2QYzu+U4bT5pZqvMrMLM/pzYMr3TLzudT08bylNLd7BlX73X5YiIdEmn4W5mQeAO4BKgFJhtZqVHtRkN/DtwjnPuFOBr3VCrZ246dwThYEBH7yKSNLpy5D4V2OCc2+ScawEeAmYd1eYm4A7nXDWAc85Xz6srzIkwe+pQnnh3B9sPNHhdjohIp7oS7oOB7R2mK+PzOhoDjDGzv5vZQjObmagCe4ubzxtJwIw7X9HRu4j0fok6oRoCRgPnA7OB35hZ/tGNzGyOmZWbWXlVVVWCVt0zBuRFuOaMITxaXqmjdxHp9boS7juAIR2mi+PzOqoE5jnnWp1zm4F1xML+PZxz9zjnypxzZQUFBSdas2fmzhhFMGD8/MX1XpciIvKBuhLui4DRZjbczNKAa4F5R7V5kthRO2bWn1g3zaYE1tkrFOVG+My0YTy+pFLXvYtIr9ZpuDvnosBcYAGwGnjEOVdhZrea2ZXxZguA/Wa2CngZ+DfnnC9H3Lr5/JFEwkFuf2Gd16WIiByXeTWkbVlZmSsvL/dk3SfrpwvW8quXNzD/qx9h/MBcr8sRkRRiZoudc2WdtdMdqifgpo+MICcS4n/+pqN3EemdFO4nIC8zzBfOHcELq/dQvuWA1+WIiLyPwv0E3TB9OIU56fz3/DV6WpOI9DoK9xOUmRbi6xeNYfHWahZU7PG6HBGR91C4n4SrTy9mZEEWP16whmhbu9fliIgcoXA/CaFggG/NHMemqnoeLt/e+Q+IiPQQhftJuqi0iDNK+vCz59dzqDnqdTkiIoDC/aSZGd++rJR9h5q5U0MCi0gvoXBPgMlD8vmnKYO5943NGlRMRHoFhXuC/NvMsQTN+OH8NV6XIiKicE+UgXkZ3HzeSP66Yhdvb/LlsDoikkQU7gk059wRDMqLcOszq2hr141NIuIdhXsCZaQFueXS8VTsrOPBd7Z5XY6IpDCFe4JdMXEgZ43ox08WrGX/oWavyxGRFKVwTzAz49ZZp1DfHOXHz631uhwRSVEK924wuiiHG6cP5+Hy7SzeWu11OSKSghTu3eQrHx3NgNwI331ypcadEZEep3DvJtnpIb57eSmrdtXx+ze3eF2OiKQYhXs3uvTUAcwYV8j//G2d7lwVkR6lcO9GZsZ/fGwCZvDdp1bqoR4i0mMU7t1scH4G37x4LK+sreLp5bu8LkdEUoTCvQdcf3YJk4rzuPXpCqrrW7wuR0RSgMK9BwQDxg+vmkhNQys/eLrC63JEJAUo3HvI+IG5zJ0xiqeW7mRBxW6vyxERn1O496AvXzCK0oG5fPuJleqeEZFupXDvQeFggJ9ePYmahhZ1z4hIt1K497DSQf/onpm/QlfPiEj3ULh74MsXjGJScR7//sQK9tQ1eV2OiPiQwt0D4WCA266ZTFNrG998dBnterCHiCSYwt0jIwuy+c5lpby+fh9/eGuL1+WIiM8o3D30qTOHMmNcIf89fw1rdx/0uhwR8ZEuhbuZzTSztWa2wcxu+YB2V5mZM7OyxJXoX2bGj66aSG4kzNw/L6Gxpc3rkkTEJzoNdzMLAncAlwClwGwzKz1Guxzgq8DbiS7Szwpy0rn9mslsqDrED+bp8kgRSYyuHLlPBTY45zY551qAh4BZx2j3H8CPAF3+8SFNH92fL50/kofLt/PU0h1elyMiPtCVcB8MbO8wXRmfd4SZnQYMcc79NYG1pZSvXziGsmF9+D+Pr2DzvnqvyxGRJHfSJ1TNLADcBnyjC23nmFm5mZVXVVWd7Kp9JRQM8IvZUwiHAnzxT4tpaIl6XZKIJLGuhPsOYEiH6eL4vMNygAnAK2a2BZgGzDvWSVXn3D3OuTLnXFlBQcGJV+1Tg/Iz+MW1U1i75yDffkIP9xCRE9eVcF8EjDaz4WaWBlwLzDu80DlX65zr75wrcc6VAAuBK51z5d1Ssc+dO6aAr184hife3cGfFm71uhwRSVKdhrtzLgrMBRYAq4FHnHMVZnarmV3Z3QWmorkXjGLGuEJufWYVi7dWe12OiCQh8+qrf1lZmSsv18H98dQ2tHLFr96gsbWNp+dOZ0BexOuSRKQXMLPFzrlO7yXSHaq9VF5mmHuvL6OhOcqcP5bT1KobnESk6xTuvdiYohxuv3YKK3bU8q3HlusEq4h0mcK9l7uotIhvXjyWp5bu5M5XNnpdjogkiZDXBUjnvnT+SNbtOchPFqxlaN9Mrpg0yOuSRKSXU7gnATPjx5+YyM6aRr7x6DIG5kUoK+nrdVki0oupWyZJpIeC3POZMgbnZ3DT/eVs0RAFIvIBFO5JpE9WGr/73BkAXP+7d6g62OxxRSLSWynck0xJ/yzu+9wZ7K1r5nO/e4eDTa1elyQivZDCPQmdNrQPd376NNbsPsjNf1pMc1TXwIvIeynck9QFYwv58VUT+fuG/Xz94aVE29q9LklEehFdLZPErjq9mOqGFv7zr6uJhJfz009MIhAwr8sSkV5A4Z7kPv+RETS0tHHb8+vICAf5z49NwEwBL5LqFO4+8JUZo2hoaePuVzeSEQ7y7cvGK+BFUpzC3QfMjG/NHEtTaxv3vrEZQAEvkuIU7j5hZnz/ilIA7n1jMw74jgJeJGUp3H3kcMCbwX1vbKat3cWnFfAiqUbh7jNmxvcuLyVgxn1vbKaptY3/9/FTCeoqGpGUonD3ITPjO5eNJystyC9e2sCh5ii3fXIyaSHd1iCSKhTuPmVm/OvFY8lKD/Hf89dQ3xzlzk+dTkZa0OvSRKQH6FDO575w3kj+6+On8uq6Kq67dyEH6lu8LklEeoDCPQVcd+ZQ7vzU6VTsrOMTd73J9gMNXpckIt1M4Z4iZk4YwAOfP5N9h5r5p7veZHlljdcliUg3UrinkDNK+vLYF88mLRjgk79+i+dW7vK6JBHpJgr3FDO6KIcnv3wO4wbkcvOflnD3qxtxznldlogkmMI9BRXkpPPQnGlcNnEgP5y/hn99ZBlNrRoTXsRPdClkioqEg/zy2imMLcrhtufXsbHqEL/+zOkMzMvwujQRSQAduaewQMD4l4+O5jefLWNTVT1X/PLvLNy03+uyRCQBFO7CRaVFPPGls8nNCHHdbxZy1yvqhxdJdgp3AWInWufNnc4lpw7kR8+t4ab7F1PToBueRJKVwl2OyE4P8avZU/je5aW8snYvl/78dRZtOeB1WSJyAhTu8h5mxg3Th/PYF88mFAxwza/f4hcvrqetXd00IsmkS+FuZjPNbK2ZbTCzW46x/F/NbJWZLTezF81sWOJLlZ40aUg+f/2X6VwxaRC3Pb+Oa379Ftv2a9gCkWTRabibWRC4A7gEKAVmm1npUc3eBcqccxOBvwA/TnSh0vNyImFuv2YyP7tmEmv3HOSSn7/Gw4u26WSrSBLoypH7VGCDc26Tc64FeAiY1bGBc+5l59zhw7qFQHFiyxSvmBkfn1LMc187l4nF+XzrsRXc8PtF7Kpt9Lo0EfkAXQn3wcD2DtOV8XnHcyMw/1gLzGyOmZWbWXlVVVXXqxTPDc7P4IHPn8n3ryhl4aYDXHzbazz0jo7iRXqrhJ5QNbNPA2XAT4613Dl3j3OuzDlXVlBQkMhVSw8IBIx/Pmc4z33tI5wyOJdbHl/Bdb95m01Vh7wuTUSO0pVw3wEM6TBdHJ/3HmZ2IfBt4ErnXHNiypPeaFi/LP78+Wn818dPZeXOWmbe/jo/f2E9zVGNTyPSW3Ql3BcBo81suJmlAdcC8zo2MLMpwK+JBfvexJcpvU0gYFx35lBe/MZ5XHxKET97YR0zb3+dV9Zq94v0Bp2Gu3MuCswFFgCrgUeccxVmdquZXRlv9hMgG3jUzJaa2bzj/DrxmcKcCL+67jT+cMNUAD73u0XMub9cT3sS8Zh5dUKsrKzMlZeXe7Ju6R7N0Tbue2Mzv3xxA23tjhumD+fLF4wkJxL2ujQR3zCzxc65ss7a6Q5VSZj0UJAvnT+Kl795PpdPGsjdr27k/J+8wh8XbqW1rd3r8kRSisJdEm5AXoTbPjmZeXPPYWRhNt99ciUX/+w1nlm+k3YNYyDSIxTu0m0mFufz8Jxp3Hd9GeGgMffP73LFr97gxdV7dH28SDdTuEu3MjM+Or6I+V89l59ePYm6plZu/EM5H7/zTV5dV6WQF+kmOqEqPaq1rZ2/LK7kly+uZ2dtE5OK85g7YzQXji/EzLwuT6TX6+oJVYW7eKIl2s5jSyq585UNbD/QyLgBOXzhvBFcPnEQ4aC+UIocj8JdkkK0rZ2nlu7k7lc3sn7vIQblRbhh+nCuOWOILqEUOQaFuySV9nbHK+v2cvcrm3hnywGy00NcXVbM584uYVi/LK/LE+k1FO6StJZtr+F3f9/MM8t30eYc540p4LNnDeO8MYUEA+qXl9SmcJekt6euiQfe3saD72yj6mAzxX0yuPaMIVxdNoSi3IjX5Yl4QuEuvtHa1s7fKvbwwNtbeXPjfoIB44KxhXyyrJgLxhXqBKyklK6Ge6gnihE5GeFggMsmDuSyiQPZsq+eh8u385fFlbyweg/9stK4cvIg/mlKMRMG5+pySpE4HblLUoq2tfPa+qpYyK/aS0tbOyMKsvjY5MFcMWkQw/vrJKz4k7plJGXUNrTy7MpdPPnuDt7efACAUwblcvnEQVx66gBdbSO+onCXlLSzppFnV+zimeW7WLq9BoBxA3KYOWEAF5cOYPzAHHXdSFJTuEvKq6xu4LmVu1lQsZvyrdU4F3vQ90WlRcwYV8iZI/qSHgp6XabIh6JwF+lg78EmXlq9lxdW7+H19ftojraTmRbk7JH9OX9sAeeOLmBov0yvyxTplMJd5DgaW9pYuGk/L63Zy8tr91JZ3QhASb9Mpo/uzzkj+3PWyH7kZ6Z5XKnI+yncRbrAOcfmffW8tq6K19bv4+1N+6lvacMMSgfmMm1EP6aN6MfUkr7kZWqsG/Gewl3kBLS2tbNsew1/37CfhZv2s3hbNS3R2CMCxxblcMbwPpxR0pfThvahuE+GTs5Kj1O4iyRAU2sby7bX8M7mAyzaWs2SrdUcao4CUJCTzmlD85k8pA+ThuQxsTif7HTdFyjdS3eoiiRAJBzkzBH9OHNEPyB289TaPQdZsq2GJVureXdbNQsq9gBgBiMLspk4OI8J8df4gTkaulg8oSN3kZNUXd/C0soalm2vYeWOWpZV1lJ1sPnI8pJ+mYwfmHvkNW5ADoPzMwhohEs5ATpyF+khfbLSuGBsIReMLTwyb09dExU7a6nYUUfFzjpW7apj/srdR5ZnpQUZXZTDmKJsRhfmMLoom5EF2Qp9SRiFu0g3KMqNUJQbYca4oiPzDjVHWbv7IGt3H2TdnoOs2V3HS2v28kh55ZE2kXCAEf2zGV6Qxcj+WQwvyKKkX+zVJ0uXZkrXKdxFekh2eojTh/Xh9GF93jP/QH0LG/YeYmPVoSPvK3fUMn/FLto79JrmZYQZ1i+TIX0zGdY39j6kTyZD+mYwKD9DQx/LeyjcRTzWNyuNqcP7MnV43/fMb4m2s+1APVv2NbBlfz1b9tezdX8DFTtqWbByN9EOyW8GRTkRBvfJYHB+BgPzI7H3vAwG5kUYmBehb1aaLt1MIQp3kV4qLRRgVGEOowpz3rcs2tbO7romth9oZHt1A5XVjeysaWRHdSNLt9fw3MomWtra3/v7ggEKc9Mpyo0wIDdCQU46hbnpFOZEKMxJpyAnnf7Z6fTNStPjDH1A4S6ShELBAMV9Minuk8lZ9Hvf8vZ2x776ZnbVNLGrtondtY3sqmtiT20Te+qaWb2rjlfXNR+5Zr+jgMW+TfTPTqdfdhr9smLvfTPT6Bt/75OVRt+sNPIzw/TJTFOXUC+kcBfxoUDA4kfkESYNOX67+uYoew82s+9QM1UHm9lb18T++pb4dAsH6ptZVl3D/kMtx/xDcFh2eoj8zHDslZFGXmaYvIwwuZH4e0aI3EiY3IwwuZEQOZF/vEfCAXUXdYMuhbuZzQR+DgSBe51zPzxqeTpwP3A6sB+4xjm3JbGlikiiZaWHGJ4e6tKTq5qjbVTXt7K/vpmahlYO1LdQ09BCdUMr1Q0t1DS0UtPQQm1jKztrG6lrbKW2sZXWtg++lyYUMLIjIbLTY6+cSIis9NgrOy3+nh4kMz1EVlqQzLQQWelBMtJCZKYFyQgHyYzPz0iLfdY3iS6Eu5kFgTuAi4BKYJGZzXPOrerQ7Eag2jk3ysyuBX4EXNMdBYuIN9JDQQbkBRmQF+nyzzjnaGxto64xSl1TK3WNrdQ1tXKwKUpdU5SDTa0caopysCnKoeb4qynK/kMtbNvfwKHmKPXNURpa2/gw91uGAkZGOEgkHv6RcICMcJD0cJBIOEgkFIi9h2Pv6aF/vKeHgqSFArHP4QBpwdh0WihAWjBwZNnh6fDh+fFl4aARDJjn30a6cuQ+FdjgnNsEYGYPAbOAjuE+C/hB/PNfgF+ZmTmvbn8VkV7BzMhMC5GZFvpQfxSO1t7uaIq2cag5SmNLG/XNbTS0RGloaaOhpY3G1tjnxsOv1virpY2m1jaaWttpbI19rm1sZU9LG83RNpqj7TS1/uO9PYGJlRaMBX04FCAcDBAOxD6HAsbXLhzDFZMGJW5lx9CVcB8MbO8wXQmcebw2zrmomdUC/YB9iShSRFJbIPCPPxLdKdrWTlO0nZZoeyz8W9tpaWuPv8f+CMSWtdPaFnu1RNtpaXOx92g70cPz21yHz+20Hpl25PfA8NE9ekLVzOYAcwCGDh3ak6sWEelUKBggOxiAdK8rOXldOeuwA+h4vr04Pu+YbcwsBOQRO7H6Hs65e5xzZc65soKCghOrWEREOtWVcF8EjDaz4WaWBlwLzDuqzTzg+vjnTwAvqb9dRMQ7nXbLxPvQ5wILiF0K+VvnXIWZ3QqUO+fmAfcBfzSzDcABYn8ARETEI13qc3fOPQs8e9S873X43ARcndjSRETkROlKfxERH1K4i4j4kMJdRMSHFO4iIj7k2QOyzawK2HqCP96f1Lz7NRW3OxW3GVJzu1Nxm+HDb/cw51ynNwp5Fu4nw8zKu/L0b79Jxe1OxW2G1NzuVNxm6L7tVreMiIgPKdxFRHwoWcP9Hq8L8EgqbncqbjOk5nan4jZDN213Uva5i4jIB0vWI3cREfkASRfuZjbTzNaa2QYzu8XrerqDmQ0xs5fNbJWZVZjZV+Pz+5rZ82a2Pv7ex+taE83Mgmb2rpk9E58ebmZvx/f3w/GRSX3FzPLN7C9mtsbMVpvZWSmyr78e//e90sweNLOI3/a3mf3WzPaa2coO8465by3mF/FtX25mp53MupMq3Ds8z/USoBSYbWal3lbVLaLAN5xzpcA04Mvx7bwFeNE5Nxp4MT7tN18FVneY/hHwM+fcKKCa2PN6/ebnwHPOuXHAJGLb7+t9bWaDgX8BypxzE4iNOHv4+ct+2t+/B2YeNe94+/YSYHT8NQe462RWnFThTofnuTrnWoDDz3P1FefcLufckvjng8T+sw8mtq1/iDf7A/AxbyrsHmZWDFwG3BufNmAGsefygj+3OQ84l9iw2TjnWpxzNfh8X8eFgIz4A34ygV34bH87514jNgx6R8fbt7OA+13MQiDfzAae6LqTLdyP9TzXwR7V0iPMrASYArwNFDnndsUX7QaKPCqru9wO/G+gPT7dD6hxzkXj037c38OBKuB38e6oe80sC5/va+fcDuCnwDZioV4LLMb/+xuOv28Tmm/JFu4pxcyygceArznn6jouiz/pyjeXOpnZ5cBe59xir2vpYSHgNOAu59wUoJ6jumD8tq8B4v3Ms4j9cRsEZPH+7gvf6859m2zh3pXnufqCmYWJBfsDzrnH47P3HP6aFn/f61V93eAc4Eoz20Ksu20Gsb7o/PjXdvDn/q4EKp1zb8en/0Is7P28rwEuBDY756qcc63A48T+Dfh9f8Px921C8y3Zwr0rz3NNevG+5vuA1c652zos6vis2uuBp3q6tu7inPt351yxc66E2H59yTklP+0NAAAA6klEQVT3KeBlYs/lBZ9tM4Bzbjew3czGxmd9FFiFj/d13DZgmpllxv+9H95uX+/vuOPt23nAZ+NXzUwDajt033x4zrmkegGXAuuAjcC3va6nm7ZxOrGvasuBpfHXpcT6oF8E1gMvAH29rrWbtv984Jn45xHAO8AG4FEg3ev6umF7JwPl8f39JNAnFfY18H+BNcBK4I9Aut/2N/AgsXMKrcS+pd14vH0LGLGrATcCK4hdSXTC69YdqiIiPpRs3TIiItIFCncRER9SuIuI+JDCXUTEhxTuIiI+pHAXEfEhhbuIiA8p3EVEfOj/A7Sw+n5wH1zgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Learning rate warmup by 10 epochs.\n",
    "lambda1 = lambda epoch: 0.95 ** epoch\n",
    "scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda1)\n",
    "\n",
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(100):\n",
    "    x.append(epoch)\n",
    "    y.append(optimizer.param_groups[0]['lr'])\n",
    "    scheduler.step()\n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Warmup "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.optim.lr_scheduler import _LRScheduler\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "\n",
    "\n",
    "class GradualWarmupScheduler(_LRScheduler):\n",
    "    \"\"\" Gradually warm-up(increasing) learning rate in optimizer.\n",
    "    Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'.\n",
    "    Args:\n",
    "        optimizer (Optimizer): Wrapped optimizer.\n",
    "        multiplier: target learning rate = base lr * multiplier\n",
    "        warmup_epoch: target learning rate is reached at warmup_epoch, gradually\n",
    "        after_scheduler: after target_epoch, use this scheduler(eg. ReduceLROnPlateau)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, optimizer, multiplier, warmup_epoch, after_scheduler=None):\n",
    "        self.multiplier = multiplier\n",
    "        if self.multiplier <= 1.:\n",
    "            raise ValueError('multiplier should be greater than 1.')\n",
    "        self.warmup_epoch = warmup_epoch\n",
    "        self.after_scheduler = after_scheduler\n",
    "        self.finished = False\n",
    "        super(GradualWarmupScheduler, self).__init__(optimizer)\n",
    "\n",
    "    def get_lr(self):\n",
    "        if self.last_epoch > self.total_epoch:\n",
    "            if self.after_scheduler:\n",
    "                if not self.finished:\n",
    "                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n",
    "                    self.finished = True\n",
    "                return self.after_scheduler.get_lr()\n",
    "            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n",
    "\n",
    "        return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.warmup_epoch + 1.) for base_lr in self.base_lrs]\n",
    "\n",
    "    def step_ReduceLROnPlateau(self, metrics, epoch=None):\n",
    "        if epoch is None:\n",
    "            epoch = self.last_epoch + 1\n",
    "        self.last_epoch = epoch if epoch != 0 else 1  # ReduceLROnPlateau is called at the end of epoch, whereas others are called at beginning\n",
    "        if self.last_epoch <= self.warmup_epoch:\n",
    "            warmup_lr = [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]\n",
    "            for param_group, lr in zip(self.optimizer.param_groups, warmup_lr):\n",
    "                param_group['lr'] = lr\n",
    "        else:\n",
    "            if epoch is None:\n",
    "                self.after_scheduler.step(metrics, None)\n",
    "            else:\n",
    "                self.after_scheduler.step(metrics, epoch - self.total_epoch)\n",
    "\n",
    "    def step(self, epoch=None, metrics=None):\n",
    "        if type(self.after_scheduler) != ReduceLROnPlateau:\n",
    "            if self.finished and self.after_scheduler:\n",
    "                if epoch is None:\n",
    "                    self.after_scheduler.step(None)\n",
    "                else:\n",
    "                    self.after_scheduler.step(epoch - self.warmup_epoch)\n",
    "            else:\n",
    "                return super(GradualWarmupScheduler, self).step(epoch)\n",
    "        else:\n",
    "            self.step_ReduceLROnPlateau(metrics, epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 500)\n",
    "scheduler = GradualWarmupScheduler(optimizer, multiplier=8, warmup_epoch=10, after_scheduler=scheduler_cosine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11839eb50>]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VdW5+PHvmzlhJglTwhBmgzKGMEgpzmjVOIAEJ1QsRXHo/MDtvb29tvZXa+vQigOT4gQixZqrVKwFZ0kIgzJGwwwyhHlMyEne3x9ng7lpIIdM+5yz38/z8HDO2mufvAvjes9aa++9RFUxxhhjItwOwBhjTHCwhGCMMQawhGCMMcZhCcEYYwxgCcEYY4zDEoIxxhjAEoIxxhiHJQRjjDGAJQRjjDGOKLcDOB9JSUnaqVMnt8MwxpiQsXz58n2qmhxI3ZBKCJ06dSI/P9/tMIwxJmSIyNZA69qUkTHGGMASgjHGGIclBGOMMYAlBGOMMQ5LCMYYY4AAE4KIjBSRAhEpFJHJVRyPFZE3nOO5ItKpwrEpTnmBiFxVofwnIrJWRNaIyBwRiauLBhljjKmZahOCiEQCU4GrgXRgrIikV6o2Hjioql2BJ4HHnHPTgWygFzASeFZEIkUkBXgIyFDVC4FIp54xxhiXBHIfQiZQqKqbAERkLpAFrKtQJwv4jfN6PvCMiIhTPldVS4DNIlLofN4252fHi0gpkAB8W/vmnJ8PC/ayavshoiMjiI4UoiIiiI+JpHl8NM0SomkeH0PzhGgSG8cQGxXZ0OEZY0yDCiQhpADbK7zfAQw6Wx1V9YnIYSDRKV9a6dwUVf1CRP6EPzGcBN5X1fer+uEiMgGYANChQ4cAwg3cf+esZev+EwHVTW4SS0rzeFJaxJPaPJ5OSY3o3roJ3Vs3pklcdJ3GZYwxbnDlTmURaYF/9JAGHALeFJHbVfXVynVVdRowDSAjI0PrMo7jJWVkD2zP/2T1wlemlJaVc+JUGYdPlnLoRCmHT57i4IlSio6WsPPgSXYeOsm6b4/wz3V7OOUrP/M5Kc3j6dGmCRemNKN/h+b0a9+CZgmWJIwxoSWQhLATaF/hfapTVlWdHSISBTQD9p/j3MuBzapaBCAiC4ChwL8lhPpUUlpGXHQksVGRxDr/Es0ToF3z+HOeV16u7Dx0koLdRynYc9T/9+6jfFiwl3InZXVt1Zj+HZozpEsiF3dNolUTWzM3xgS3QBLCMqCbiKTh78yzgVsr1ckBxgFfAKOAxaqqIpIDvC4iTwDtgG5AHlAODBaRBPxTRpcBDf6QomKfPyGcr4gIoX3LBNq3TODy9NZnyo+V+Phq+yFWbDvIim2HeH/dHubl7wCgZ5smDO+ezPe6JZGZ1tLWJIwxQafahOCsCTwALMJ/NdAsVV0rIo8A+aqaA8wEXnEWjQ/gXDHk1JuHfwHaB0xS1TIgV0TmAyuc8pU400INpaxcKS1T4muQEM6mcWwUQ7smMbRrEuAfSaz99gifFBbxydf7eOmzLUz7eBNNYqO49IJWjOzVhu/3SCYhJqSeMWiMCVOiWqfT8vUqIyND6+ppp8dLfPT670VMubonP/p+lzr5zOqcOOVj6ab9vLdmN/9ct4eDJ0qJi47g+92TubFfCpf0bGUjB2NMnRKR5aqaEUhdz341LS4tA6jRlFFNJcREcWnP1lzaszW+snLythxg0ZrdLFyzm0Vr99A8IZrrerfj5gGp9Elthv/KXWOMaRjeTQjOVUJx0e48vSMqMoKhXZIY2iWJ/7o2nU8L97FgxU7m5W/nlaVb6dqqMbcN6sDNA1Jpape1GmMagHcTggsjhLOJioxgRI9WjOjRiiPFpfxj9S7m5G3nf/53HX98r4Ab+qVw55COXNC2qduhGmPCmOcTQrDN2TeNi2bMwA6MGdiBNTsP88oXW3lr5Q7m5G0js1NLJgzvzKU9WxERYdNJxpi65dmnnRaXujtlFIgLU5rx2Kje5E65nP/8wQXsPHSSe1/O56qnPubN/O3/5+Y4Y4ypreDtDetZSRBNGVWnWUI0936vMx/9YgRPZ/clKjKCX8z/iuF/XMKMTzadGe0YY0xteDYhFPtCJyGcFhUZQVbfFBY+NIzZ92SSltSI3727nu/9cQkvfbbZEoMxpla8mxBCYMrobESE73dPZs6EwbwxYTCdkxrxm/9dxyV/+pBXl261qSRjTI2EXm9YR85cZRRki8rna1DnROZOGMzr9w6iXfN4/vPva7j0zx/y9qqdlJeHzk2Hxhj3eTghnB4hhHZCAP+IYWjXJOZPHMLsezJpnhDNw3NXceOzn7FsywG3wzPGhAgPJ4TTawjh809weiopZ9IwnrilD3uOlDD6+S+Y+Mpytuw77nZ4xpggFz694XkKxUXlQEVECDf1T2XJz0fwsyu68/E3RVzx5Ec8+u46jpX43A7PGBOkvJsQnCmj2Kjw/SeIj4nkwcu68eEvRnBz/1RmfLqZy/78Ie989S2h9FBDY0zDCN/esBolpWXERkV44gFyrZrE8Yebe/PW/ReT3CSWB15fyR0z89hYdMzt0IwxQcSzCaG4tGab44Syvu2b8/akYTyS1Ysvdxxi5FMf8/iiDXb/gjEGCDAhiMhIESkQkUIRmVzF8VgRecM5nisinSocm+KUF4jIVU5ZDxFZVeHPERH5cV01KhDFpeVhtaAcqMgI4c4hnVj8sxFc16cdU5ds5OqnPyF30363QzPGuKzaHlFEIoGpwNVAOjBWRNIrVRsPHFTVrsCTwGPOuen4d0/rBYwEnhWRSFUtUNW+qtoXGACcAN6qozYFpKbbZ4aL5CaxPHFLX16/dxC+8nLGTFvKf/19jS06G+NhgXxFzgQKVXWTqp4C5gJZlepkAbOd1/OBy8Q/OZ8FzFXVElXdDBQ6n1fRZcBGVd1a00bURHFpWcjflFYXhnZNYtGPh3PPxWm8mruVq578mI++LnI7LGOMCwJJCCnA9grvdzhlVdZRVR9wGEgM8NxsYE7gIdcNr04ZVSUhJopfX5fO/IlDiY+JZNysPH7+5pccLS51OzRjTANytUcUkRjgeuDNc9SZICL5IpJfVFR331yLS8uI9fCUUVUGdGzBuw8NY9IlXViwYgcjn/qEvM12p7MxXhFIQtgJtK/wPtUpq7KOiEQBzYD9AZx7NbBCVfec7Yer6jRVzVDVjOTk5ADCDUyxr9zTawhnExsVyS+u6smbE4cSFSmMmfYFf/jHBkp8diWSMeEukISwDOgmImnON/psIKdSnRxgnPN6FLBY/Xc+5QDZzlVIaUA3IK/CeWNxYboI/PchxIXxTWm1NaBjCxY+9D2yB7bn+Y82csPUz/l6z1G3wzLG1KNqe0RnTeABYBGwHpinqmtF5BERud6pNhNIFJFC4KfAZOfctcA8YB3wHjBJVcsARKQRcAWwoG6bFBgv3odwvhrFRvH/burN9Dsz2HukmGv/+ikzP91sdzkbE6YC2lNZVRcCCyuV/brC62Jg9FnOfRR4tIry4/gXnl1hi8qBuyK9Nf06DGfy377it++s4/PCffxpdB9aNIpxOzRjTB3ybI/o9fsQzldS41im35nBb65L55Nv9nHNXz4h3x6tbUxY8W5CsCmj8yYi3HVxGn+7byjRkRGMmbaUqUsKbSMeY8KEJxOCqvqnjGxRuUYuSm3GOw8NY+SFbXh8UQHjXsxj37ESt8MyxtSSJ3vEEmfPYbsPoeaaxkXzzNh+/P7Gi8jdfIBrnv6E5VttCsmYUObNhBBG22e6SUS4dVAH3p50MfExkYx5YSkvf7HFrkIyJkR5MiF8t1uaJ5tf5y5o25ScB4YxvHsyv357LT9780t7pLYxIciTPeKZ/ZTt4XZ1pll8NDPuzOAnl3fnrZU7uenZz9l+4ITbYRljzoNHE4JNGdWHiAjh4cu7MWvcQHYcPMG1f/2UDwv2uh2WMSZAHk0INmVUny7p2Yr/fXAYbZvFcfdLy5i6pNDWFYwJAZ7sEb9LCDZCqC8dExvx1v0Xc13vdjy+qICH566ydQVjglxAj64IN8W+01NGnsyHDSY+JpKns/vSo00THl9UwNYDJ5h+xwBaNY1zOzRjTBU82SOe/qYaa4vK9U5EmHRJV164YwDf7DnK9c98xuodh90OyxhTBU8nBJsyajhX9WrD/IlDiRAY/cLnvPvVLrdDMsZU4smE8N2NaZ5svmvS2zXl7QeG0atdMya9voKnPvjanoNkTBDxZI/43Y1pNkJoaMlNYnn9h4O4qX8KT33wDT9+Y5XtxmZMkPDmorJNGbkqNiqSP4/uQ9dWjfnjewXsPlLMtDsG0DzB9lcwxk0BjRBEZKSIFIhIoYhMruJ4rIi84RzPFZFOFY5NccoLROSqCuXNRWS+iGwQkfUiMqQuGhSIk6ecKSN72qlrRIT7R3Tl6ey+rNp2iJue+5xt++3OZmPcVG2PKCKRwFTgaiAdGCsi6ZWqjQcOqmpX4EngMefcdPx7MPcCRgLPOp8H8DTwnqr2BPrg356zQRT7yoiKEKIiLSG4LatvCq+Mz2T/sVPc9NxnrNp+yO2QjPGsQHrETKBQVTep6ilgLpBVqU4WMNt5PR+4TETEKZ+rqiWquhkoBDJFpBkwHP9ezKjqKVVtsJ7ANscJLoM6J/K3+4YSHxNJ9rQvWLR2t9shGeNJgSSEFGB7hfc7nLIq66iqDziMf7/ks52bBhQBL4rIShGZISKNatSCGrD9lINP11aNeev+i+nRpikTX13OrE83ux2SMZ7jVq8YBfQHnlPVfsBx4N/WJgBEZIKI5ItIflFRUZ388JLSMrspLQglNY5l7g8Hc8UFrXnknXX89p11dlmqMQ0okISwE2hf4X2qU1ZlHRGJApoB+89x7g5gh6rmOuXz8SeIf6Oq01Q1Q1UzkpOTAwi3esW+MhshBKn4mEieu30Adw3txMxPN/OTeas45TxqxBhTvwLpFZcB3UQkTURi8C8S51SqkwOMc16PAhar//GWOUC2cxVSGtANyFPV3cB2EenhnHMZsK6WbQmYf8rIRgjBKjJC+O/r0vnlyB68vepb7n05n+MlPrfDMibsVXsfgqr6ROQBYBEQCcxS1bUi8giQr6o5+BeHXxGRQuAA/qSBU28e/s7eB0xS1dN3IT0IvOYkmU3A3XXctrOyReXgd/qy1MRGMUxZsJpbZ+Ty4l0DadnI7lUwpr5IKD2nPiMjQ/Pz82v9OTc9+xnxMZG8du/gOojK1Lf31+7mwTkrSWkRz8v3ZJLaIsHtkIwJGSKyXFUzAqnryYn04tJy2z4zhFzZqw2vjB9E0dESRj33BQW7j7odkjFhyZsJwVdGXIwlhFCSmdaSNycOoVyV0c9/Tv6WA26HZEzY8WRCKLERQkjq2aYpf7tvKEmNY7ltRi7/Wr/H7ZCMCSueTAj+RWVPNj3ktW+ZwJsTh9CjTRN+9Mpycr781u2QjAkbnuwV7Sqj0JbYOJbX7h1E/44teHjuSubkbXM7JGPCgjcTgs8eXRHqmsRFM/vuTIZ3S2bKgtXM+GST2yEZE/I81yuWlpVTVq62hhAG4mMimX5nBtdc1Ibfvbuepz74mlC6jNqYYOO5DXJsc5zwEhMVwV+y+5EQs5qnPviGY8U+fvWDC/A/bNcYcz48mBBsP+VwExUZwR9v7k3j2ChmfLqZYyU+Hr3xIiIjLCkYcz48mBD8I4RYGyGElQjn+UdN4qL46+JCjpX4eHJMX6JtEyRjAua5hHB6Q3ebMgo/IsLPruxBo9go/vCPDZw8VcbU2/rbf2tjAuS5r09npoxsP+WwNfH7XfjdDReyuGAvP3w5n5Onyqo/yRjjxYRgIwQvuH1wR/54c28+LdzHPS8t48Qpe3y2MdXxYEI4vahsCSHcjc5oz5O39CV3837umrWMY7angjHn5MGEcHqE4Lmme9IN/VL4y9h+LN92kDtn5nKkuNTtkIwJWp7rFYttUdlzru3djqm39mf1zsPcPiOXwycsKRhTlYASgoiMFJECESkUkclVHI8VkTec47ki0qnCsSlOeYGIXFWhfIuIrBaRVSJS+11vAvTdorIlBC8ZeWEbnr99ABt2HWXs9KUcOH7K7ZCMCTrVJgQRiQSmAlcD6cBYEUmvVG08cFBVuwJPAo8556bj306zFzASeNb5vNMuUdW+ge7mUxdsysi7LrugNdPHZbCx6Bi3Tl/KvmMlbodkTFAJpFfMBApVdZOqngLmAlmV6mQBs53X84HLxP/sgCxgrqqWqOpmoND5PNfYjWne9v3uycy6ayBb9h8ne9pS9h4pdjskY4JGIAkhBdhe4f0Op6zKOqrqAw4DidWcq8D7IrJcRCac7YeLyAQRyReR/KKiogDCPbcSnz26wusu7prES3dn8u2hk2RPW8ruw5YUjAF3F5WHqWp//FNRk0RkeFWVVHWaqmaoakZycnKtf2hxaRkiEGOPNPC0wZ0TefmeTPYeLWHMtC/Ydfik2yEZ47pAesWdQPsK71OdsirriEgU0AzYf65zVfX033uBt2igqaTi0jLioiLtaZiGjE4teWV8JgeOnSJ72lJLCsbzAkkIy4BuIpImIjH4F4lzKtXJAcY5r0cBi9X/YPocINu5CikN6AbkiUgjEWkCICKNgCuBNbVvTvWKS21zHPOdfh1a8LKTFMba9JHxuGp7RmdN4AFgEbAemKeqa0XkERG53qk2E0gUkULgp8Bk59y1wDxgHfAeMElVy4DWwKci8iWQB7yrqu/VbdOqZttnmsr6dWjB7PGZ7Dt2iuxpX1hSMJ4lobTDVEZGhubn1+6WhQfnrGTNzsMs+fmIugnKhI0V2w5y58w8khrHMHfCENo0i3M7JGNqTUSWB3ppv+fmTopLy4i1J52aKvTv0ILZ9/hHCmOn2/SR8R7P9Yw2ZWTOZUDHFsy+ZyB7jxQzdvpS9th9CsZDPJcQSmxR2VRjQMeWvDw+k71HismeZknBeIfnesZin40QTPUGdGzJ7Hv8SWGsJQXjEd5LCM59CMZUJ6OTPynscZKCPebChDsPJgSbMjKBy+jUkpfuyWT3kWKyp1tSMOHNcz2jLSqb8zXQGSnsPmxJwYQ3SwjGBGBgp5a8dLc/KYydvpS9Ry0pmPDjvYTgKyfWpoxMDWSmteTFuwby7aFibp2eS9FR20/BhBdP9Yzl5copX7ktKpsaG9Q5kRfvHsjOgydtkx0TdjyVEL7bC8ESgqm5wZ0TmXlXBtsPnuC26bnst6RgwoSnEoJtn2nqytAuScwc59957bYZubZHswkLnuoZi32nE4KNEEztXdzVnxQ27zvOrdOXctCSgglxnkoIJ0/ZCMHUrWHdkph+Zwab9vlHCodOWFIwoctTPWNxqbOGYIvKpg4N757MtDsGULj3GLfNyOXwiVK3QzKmRryVEGzKyNSTET1a8cIdA/hmzzFun5nL4ZOWFEzoCSghiMhIESkQkUIRmVzF8VgRecM5nisinSocm+KUF4jIVZXOixSRlSLyTm0bEojTi8p2H4KpD5f0bMVzt/dnw+4j3GlJwYSgantGEYkEpgJXA+nAWBFJr1RtPHBQVbsCTwKPOeem49+DuRcwEnjW+bzTHsa/LWeDKCm1y05N/brsgtY8d9sA1u06wrhZeRwptqRgQkcgX5UzgUJV3aSqp4C5QFalOlnAbOf1fOAyERGnfK6qlqjqZqDQ+TxEJBX4ATCj9s0IzJnLTm0NwdSjy9NbM/XW/qzZeZhxs/I4aknBhIhAEkIKsL3C+x1OWZV1VNUHHAYSqzn3KeCXQPm5friITBCRfBHJLyoqCiDcs/tuDcGmjEz9urJXG565tT+rdxzmrheXcazE53ZIxlTLlZ5RRK4F9qrq8urqquo0Vc1Q1Yzk5ORa/dximzIyDWjkhW3469h+rNp+iLtm5VlSMEEvkISwE2hf4X2qU1ZlHRGJApoB+89x7sXA9SKyBf8U1KUi8moN4j8v392pbAnBNIyrL2rLX7L7sXL7Ie55cRnHLSmYIBZIQlgGdBORNBGJwb9InFOpTg4wznk9ClisquqUZztXIaUB3YA8VZ2iqqmq2sn5vMWqensdtOecvhsh2JSRaTg/6N2Wp8b0JX/rAe55aRknTllSMMGp2p7RWRN4AFiE/4qgeaq6VkQeEZHrnWozgUQRKQR+Ckx2zl0LzAPWAe8Bk1S1rO6bERhbVDZuua5PO54c05dlWw4w/qX8M3fNGxNMogKppKoLgYWVyn5d4XUxMPos5z4KPHqOz/4Q+DCQOGqr2FdGTGQEERHSED/OmP8jq28KqvDTeasYP3sZM8cNJD7GvpyY4OGpuZOSUtscx7jrhn4p/Gl0H77YtJ8fvpx/ZtRqTDDwVO9o22eaYHBT/1QeH9WHzzbus6RggooHE4KnmmyC1KgBqTx2c28+LdzHj15ZbknBBAVP9Y7FpeXE2wjBBIlbMtrzh5su4qOvi7jv1eWU+CwpGHd5KyH4bMrIBJcxAzvw/266iCUFRdz36gpLCsZV3koIpWV2yakJOmMzO/DojReyeMNeJr22glO+cz7NxZh647GEYFcZmeB026CO/PaGC/lg/V4mvW5JwbjDU72jXWVkgtkdgzvySFYv/rluDw/OWUFpmSUF07A8lRBKfOWWEExQu3NIJ/77unQWrd3DQ3NWWlIwDcpTCcG/huCpJpsQdPfFafzXten8Y81ufjx3FT5LCqaBBPToinBhU0YmVIwfloaq8rt31yMCT43pS1SkfZkx9ctjCaHcbkwzIePe73WmXJXfL9xAhAhP3NLHkoKpV55JCKpq9yGYkDNheBfKyuGx9zYQIfDnW/oSaQ9nNPXEMwnhVFk5qrY5jgk9943oQrkqjy8qIEKEx0f3saRg6oVnEsLpzXFibVHZhKBJl3RFVfnT+18TESH88ebe9hh3U+cC6h1FZKSIFIhIoYhMruJ4rIi84RzPFZFOFY5NccoLROQqpyxORPJE5EsRWSsi/1NXDTqbEts+04S4By7txk8u78785TuYvOArysvV7ZBMmKl2hCAikcBU4ApgB7BMRHJUdV2FauOBg6raVUSygceAMSKSjn+LzF5AO+ADEekOlACXquoxEYkGPhWRf6jq0jptXQXfbZ9pCcGErocv70a5Kk//6xsiRPj9jRfZSMHUmUBGCJlAoapuUtVTwFwgq1KdLGC283o+cJmIiFM+V1VLVHUzUAhkqt8xp36086dev+4U+06PEGzKyIS2H1/ejQcv7crcZdv51d/X2EjB1JlA1hBSgO0V3u8ABp2tjqr6ROQwkOiUL610bgqcGXksB7oCU1U1tyYNCJTtp2zChYjw0yu6U67K1CUbiRD43Q0X4v8OZkzNubaorKplQF8RaQ68JSIXquqayvVEZAIwAaBDhw41/nk2ZWTCiYjw8yt7UK7w3IcbiRDhkaxelhRMrQSSEHYC7Su8T3XKqqqzQ0SigGbA/kDOVdVDIrIEGAn8W0JQ1WnANICMjIwaj43PjBBsysiECRHhl1f1oLxceeHjTUQI/OZ6Swqm5gLpHZcB3UQkTURi8C8S51SqkwOMc16PAharqjrl2c5VSGlANyBPRJKdkQEiEo9/wXpD7ZtzdsV2lZEJQyLC5Kt78sPvpTH7i622pmBqpdoRgrMm8ACwCIgEZqnqWhF5BMhX1RxgJvCKiBQCB/AnDZx684B1gA+YpKplItIWmO2sI0QA81T1nfpo4GnFvtNTRjZCMOFFRPiPay4gJiqCqUs2UlJazh9H9bab18x5C2gNQVUXAgsrlf26wutiYPRZzn0UeLRS2VdAv/MNtjZOjxBibVHZhCER4RdX9SQuKpI///NrSnxlPDmmL9H27CNzHjxzp7LdmGa84MHLuhEbHcHvF26gxFfOM7f2sy9BJmCe+frw3VVGnmmy8agJw7uc2XltwsvLz4yOjamOZ3pHW1Q2XnLnkE48dvNFfPxNEXe/uIzjJT63QzIhwDsJwVdGZITYnKrxjDEDO/DELX3I3byfcbPyOFJc6nZIJsh5pncsLi237TON59zYL5Vnbu3Pqu2HuGNGLodOnHI7JBPEPNND2vaZxquuuagtz98+gPW7jjJ2ei77j5W4HZIJUh5KCOWWEIxnXZ7emhnjMti87xi3vPAFuw6fdDskE4S8kxB8ZcTaFUbGw4Z3T2b23ZnsPVLCqOe+YPO+426HZIKMZ3rIktIye9Kp8bxBnROZM2EwxaVljH7+c9Z+e9jtkEwQ8UxC8E8Zeaa5xpzVhSnNmDdxCDGREWRPW8qyLQfcDskECc/0kLaobMx3uiQ35s37hpLcOJY7ZuayZMNet0MyQcAzCeGkJQRj/o+U5vHMmziELsmN+eHL+eR8+a3bIRmXeSYh+EcInmmuMQFJahzLnAmD6d+xBQ/PXcmrS7e6HZJxkWd6SP+NaTZCMKaypnHRvHxPJpf2aMV//n0NU5cU4t/OxHiNZxJCia+MWJsyMqZKcdGRPH/HAG7o247HFxXwyDvrbKMdD/LM46/tKiNjzi06MoInbulLy0axzPpsM3uPlvDn0X1s7c1DAuohRWSkiBSISKGITK7ieKyIvOEczxWRThWOTXHKC0TkKqesvYgsEZF1IrJWRB6uqwadjV1lZEz1IiKEX1+Xzq+uuYB3v9rFuFl5HD5pD8XzimoTgrPN5VTgaiAdGCsi6ZWqjQcOqmpX4EngMefcdPzbafYCRgLPOp/nA36mqunAYGBSFZ9ZZ3xl5fjK1dYQjAnQD4d35unsvqzYdpDRz39uj7rwiEBGCJlAoapuUtVTwFwgq1KdLGC283o+cJmIiFM+V1VLVHUzUAhkquouVV0BoKpHgfVASu2bUzXbT9mY85fVN4XZd2ey61AxNz37OQW7j7odkqlngfSQKcD2Cu938O+d95k6quoDDgOJgZzrTC/1A3IDD/v82OY4xtTM0K5JvPGjIZSVK6Of/5zcTfvdDsnUI1e/MotIY+BvwI9V9chZ6kwQkXwRyS8qKqrRz/kuIdgIwZjzld6uKQvuH0qrpnHcMTOPhat3uR2SqSeB9JA7gfYV3qc6ZVXWEZEooBmw/1znikg0/mTwmqouONsPV9VpqpqhqhnJyckBhPvvvttP2UYIxtREaosE5k+DGNRHAAAOiUlEQVQcQu/UZkx6fQXTPt5o9yqEoUASwjKgm4ikiUgM/kXinEp1coBxzutRwGL1/7bkANnOVUhpQDcgz1lfmAmsV9Un6qIh53J6hBBri8rG1FjzhBhevXcQ11zUlt8v3MCUBaspLSt3OyxTh6q9D0FVfSLyALAIiARmqepaEXkEyFfVHPyd+ysiUggcwJ80cOrNA9bhv7JokqqWicgw4A5gtYiscn7Uf6jqwrpuIPhvSgObMjKmtuKiI/lrdj86JzXir4sL2X7wBM/eOoBmCdFuh2bqQEA3pjkd9cJKZb+u8LoYGH2Wcx8FHq1U9ikg5xtsTdmUkTF1JyJC+NmVPeiU2IjJC77ipuc+Y9ZdA+mY2Mjt0EwteeIrs11lZEzdu3lAKq+OH8T+46e4Yepntq9CGPBIQrD7EIypD4M6J/LW/RfTIiGG26bn8tbKHW6HZGrBEz3kmRGCLSobU+fSkhqx4P6h9O/YnJ+88SWPL9pgD8YLUd5ICD6bMjKmPjVPiOHlewYxJqM9U5ds5N6X8zlSbM9ACjXeSAg2ZWRMvYuJiuAPN1/Eb7N68fHXRdww9TM2Fh1zOyxzHjzRQ9qisjENQ0S4Y0gnXr13EIdPlHLDM5/xr/V73A7LBMgTCaGktAwRiI3yRHONcd3gzonkPDiMDokJ3PtyPs8s/sbubA4Bnughi33lxEZF4L9B2hjTEFKaxzN/4lCu79OOP73/NZNeX8GxEp/bYZlz8EZCsM1xjHFFfEwkT43py6+uuYD31uzm+mc+tcdoBzHvJAS75NQYV4gIPxzemVfvHcSRkz6ypn7KghV2v0Iw8khCsP2UjXHb0C5JLHxoGH1Sm/PTeV8yZcFXZy74MMHBE72kTRkZExxaNY3jtXsHcf+ILszJ285Nz37O1v3H3Q7LOLyREHzlxFpCMCYoREVG8MuRPZl1VwY7D53k2r98yntrbNOdYOCNhFBaRpxdcmpMULm0Z2veeXAYnZMbMfHVFfzHW6s5ecqmkNzkiV6yxKaMjAlK7Vsm8ObEofxoeGdez93GtX/9hLXfHnY7LM/yREKwRWVjgldMVARTrrmA1+4dxLESHzdO/ZwZn2yyB+S5IKBeUkRGikiBiBSKyOQqjseKyBvO8VwR6VTh2BSnvEBErqpQPktE9orImrpoyLkU+2yEYEywu7hrEu89PJwRPZL53bvrGfdiHnuPFLsdlqdUmxBEJBKYClwNpANjRSS9UrXxwEFV7Qo8CTzmnJuOfzvNXsBI4Fnn8wBecsrqnd2HYExoaNEohhfuGMCjN17Isi0HGPn0J7z7lS04N5RARgiZQKGqblLVU8BcIKtSnSxgtvN6PnCZ+J8TkQXMVdUSVd0MFDqfh6p+jH//5XpnU0bGhA4R4bZBHXnnwWG0bxHPpNdXMOm1Few/VuJ2aGEvkF4yBdhe4f0Op6zKOqrqAw4DiQGee04iMkFE8kUkv6io6HxOPcPuQzAm9HRt1YS/3TeUX1zVg/fX7ebKJz9m4WobLdSnoP/arKrTVDVDVTOSk5Nr9BnvPvQ9xg9Lq+PIjDH1LSoygkmXdOWdB79Hu+bx3P/aCh54fQUHjp9yO7SwFEhC2Am0r/A+1Smrso6IRAHNgP0BnlvvurZqTKumcQ39Y40xdaRHmyYsuH8oP7+yO4vW7uaKJz7i7VU77ZHadSyQhLAM6CYiaSISg3+ROKdSnRxgnPN6FLBY/f+lcoBs5yqkNKAbkFc3oRtjvCQ6MoIHLu1GzgPDSG2ZwMNzV3HHzDy27LNHX9SVahOCsybwALAIWA/MU9W1IvKIiFzvVJsJJIpIIfBTYLJz7lpgHrAOeA+YpKplACIyB/gC6CEiO0RkfN02zRgTji5o25QF9w3lkaxefLn9EFc+9TF/+dc3lPjsLufaklAacmVkZGh+fr7bYRhjgsSeI8U88r/reHf1LrokN+K3N1zI0C5JbocVVERkuapmBFI36BeVjTHmbFo3jWPqbf158a6BlPjKuXV6LhNfWc62/SfcDi0kWUIwxoS8S3q24oOffp+fXdGdj74u4vInPuKx9zbYlp3nyRKCMSYsxEVH8uBl3Vjy8xFc27stz324kUv+9CHz8rdTZs9FCoglBGNMWGnTLI4nxvTlrfuHktI8nl/O/4qrn/6YRWt322Wq1bCEYIwJS/06tGDBfUN55tZ++MqUH72ynBuf/ZzPC/e5HVrQsoRgjAlbERHCtb3b8f5PhvPYzRex50gxt87I5fYZuazYdtDt8IKOXXZqjPGM4tIyXl26lWc/3MiB46cY2iWRSZd0ZWiXRPzP4ww/53PZqSUEY4znHC/xMSdvG9M+3sTeoyX0ad+cSSO6cPkFrYmICK/EYAnBGGMCUFxaxoIVO3n+o41sO3CC7q0bc/fFadzQN4X4mPB4QrIlBGOMOQ++snLe+WoXL3y8ifW7jtAsPprsge25fXBH2rdMcDu8WrGEYIwxNaCqLNtykNmfb+E95zLVS3u2ZszA9ozokUx0ZOhdh3M+CSGqvoMxxphQISJkprUkM60l3x46yWu5W3lj2Q4+WL+HpMax3Nw/hdEZqXRt1cTtUOuFjRCMMeYcSsvK+aigiHn521m8YS++cqVPajN+0LstV1/YNuinlGzKyBhj6sG+YyX8feVO3l71Lat3HgY4kxyu6tWGjomNXI7w31lCMMaYerZt/wkWrtnFu1/tOpMc0pIa8f3uyYzokczgzolBsZd7nScEERkJPA1EAjNU9Q+VjscCLwMD8G+dOUZVtzjHpgDjgTLgIVVdFMhnVsUSgjEmGG3bf4LFG/bw4ddFfLFxPyW+cmKjIsjo1IKMji0Z2Kkl/To0p1Fswy/b1mlCEJFI4GvgCmAH/i01x6rqugp17gd6q+pEEckGblTVMSKSDswBMoF2wAdAd+e0c35mVSwhGGOCXXFpGUs37eejr4vI23yA9buOUK4QGSGkt23KRanNuKBtU9LbNqVnmyb1niTq+iqjTKBQVTc5Hz4XyMK/LeZpWcBvnNfzgWfEfx94FjBXVUuAzc4Wm5lOveo+0xhjQk5cdCQjerRiRI9WABwtLmXltkPkbznAsi0HeefLb3k9dxsAItCxZQJpSY3o0DKBDon+v9u3jKdVkziax0c36J3TgSSEFGB7hfc7gEFnq6OqPhE5DCQ65UsrnZvivK7uM40xJuQ1iYtmePdkhndPBvz3Ouw8dJL1u46yftcR1u86wtb9J1i25eC/begTGSEkNoqhY2ICb04cWu+xBv19CCIyAZgA0KFDB5ejMcaY2hERUlskkNoigSvSW58pV1UOnihl24ETbD9wgqKjJew7VsL+Y6doqOfuBZIQdgLtK7xPdcqqqrNDRKKAZvgXl891bnWfCYCqTgOmgX8NIYB4jTEm5IgILRvF0LJRDH3bN3clhkDuw14GdBORNBGJAbKBnEp1coBxzutRwGL1r1bnANkiEisiaUA3IC/AzzTGGNOAqh0hOGsCDwCL8F8iOktV14rII0C+quYAM4FXnEXjA/g7eJx68/AvFvuASapaBlDVZ9Z984wxxgTKbkwzxpgwdj6XnYbeo/uMMcbUC0sIxhhjAEsIxhhjHJYQjDHGAJYQjDHGOELqKiMRKQK21vD0JGBfHYYTCqzN3mBt9oaatrmjqiYHUjGkEkJtiEh+oJdehQtrszdYm72hIdpsU0bGGGMASwjGGGMcXkoI09wOwAXWZm+wNntDvbfZM2sIxhhjzs1LIwRjjDHnEPYJQURGikiBiBSKyGS346krIjJLRPaKyJoKZS1F5J8i8o3zdwunXETkL86/wVci0t+9yGtORNqLyBIRWScia0XkYac8bNstInEikiciXzpt/h+nPE1Ecp22veE8Rh7nUfNvOOW5ItLJzfhrQ0QiRWSliLzjvA/rNovIFhFZLSKrRCTfKWvQ3+2wTggiEglMBa4G0oGxIpLublR15iVgZKWyycC/VLUb8C/nPfjb3835MwF4roFirGs+4Geqmg4MBiY5/z3Dud0lwKWq2gfoC4wUkcHAY8CTqtoVOAiMd+qPBw465U869ULVw8D6Cu+90OZLVLVvhctLG/Z3W1XD9g8wBFhU4f0UYIrbcdVh+zoBayq8LwDaOq/bAgXO6xeAsVXVC+U/wNvAFV5pN5AArMC///g+IMopP/N7jn+PkSHO6yinnrgdew3amoq/A7wUeAcQD7R5C5BUqaxBf7fDeoQApADbK7zf4ZSFq9aqust5vRs4vWFr2P07ONMC/YBcwrzdztTJKmAv8E9gI3BIVU/vyF6xXWfa7Bw/DCQ2bMR14ingl0C58z6R8G+zAu+LyHJnL3lo4N/tQPZUNiFIVVVEwvISMhFpDPwN+LGqHpEKO5CHY7vVv8tgXxFpDrwF9HQ5pHolItcCe1V1uYiMcDueBjRMVXeKSCvgnyKyoeLBhvjdDvcRwk6gfYX3qU5ZuNojIm0BnL/3OuVh8+8gItH4k8FrqrrAKQ77dgOo6iFgCf7pkuYicvoLXcV2nWmzc7wZsL+BQ62ti4HrRWQLMBf/tNHThHebUdWdzt978Sf+TBr4dzvcE8IyoJtzdUIM/r2ec1yOqT7lAOOc1+Pwz7GfLr/TuTJhMHC4wjA0ZIh/KDATWK+qT1Q4FLbtFpFkZ2SAiMTjXzNZjz8xjHKqVW7z6X+LUcBidSaZQ4WqTlHVVFXthP//2cWqehth3GYRaSQiTU6/Bq4E1tDQv9tuL6Q0wELNNcDX+Oddf+V2PHXYrjnALqAU//zhePzzpv8CvgE+AFo6dQX/1VYbgdVAhtvx17DNw/DPs34FrHL+XBPO7QZ6AyudNq8Bfu2UdwbygELgTSDWKY9z3hc6xzu73YZatn8E8E64t9lp25fOn7Wn+6qG/t22O5WNMcYA4T9lZIwxJkCWEIwxxgCWEIwxxjgsIRhjjAEsIRhjjHFYQjDGGANYQjDGGOOwhGCMMQaA/w9GL8WTg7MTfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "x = list()\n",
    "y = list()\n",
    "for epoch in range(500):\n",
    "    x.append(epoch)\n",
    "    y.append(scheduler.get_lr()[0])\n",
    "    scheduler.step() \n",
    "plt.plot(x,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. 对不同层设置不同的学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv5_params = list(map(id, net.conv5.parameters()))\n",
    "conv4_params = list(map(id, net.conv4.parameters()))\n",
    "base_params = filter(lambda p: id(p) not in conv5_params + conv4_params,\n",
    "                     net.parameters())\n",
    "\n",
    "optimizer = torch.optim.SGD([\n",
    "            {'params': base_params},\n",
    "            {'params': net.conv5.parameters(), 'lr': lr * 100},\n",
    "            {'params': net.conv4.parameters(), 'lr': lr * 100}],\n",
    "            , lr=lr, momentum=0.9)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
